From 89bef9bea071c7e29b4b7cd6392897f596ae2c03 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 13 Jul 2023 15:45:35 +0200 Subject: [PATCH 001/109] Add entrypoint v2 --- exegol/config/ConstantConfig.py | 19 ++-- exegol/model/ExegolContainer.py | 5 ++ exegol/utils/DockerUtils.py | 51 ++++++----- exegol/utils/entrypoint/EntrypointUtils.py | 30 +++++++ exegol/utils/entrypoint/__init__.py | 0 exegol/utils/entrypoint/entrypoint.sh | 100 +++++++++++++++++++++ setup.py | 4 + 7 files changed, 179 insertions(+), 30 deletions(-) create mode 100644 exegol/utils/entrypoint/EntrypointUtils.py create mode 100644 exegol/utils/entrypoint/__init__.py create mode 100644 exegol/utils/entrypoint/entrypoint.sh diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index 1b3c7618..43e3087a 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -15,6 +15,8 @@ class ConstantConfig: # Path of the Dockerfile build_context_path_obj: Path build_context_path: str + # Path of the Entrypoint + entrypoint_context_path_obj: Path # Exegol config directory exegol_config_path: Path = Path().home() / ".exegol" # Docker Desktop for mac config file @@ -33,12 +35,11 @@ class ConstantConfig: EXEGOL_RESOURCES_REPO: str = "https://github.com/ThePorgs/Exegol-resources.git" @classmethod - def findBuildContextPath(cls) -> Path: - """Find the right path to the build context from Exegol docker images. + def findResourceContextPath(cls, resource_folder: str, source_path: str) -> Path: + """Find the right path to the resources context from Exegol package. Support source clone installation and pip package (venv / user / global context)""" - dockerbuild_folder_name = "exegol-docker-build" - local_src = cls.src_root_path_obj / dockerbuild_folder_name - if local_src.is_dir(): + local_src = cls.src_root_path_obj / source_path + if local_src.is_dir() or local_src.is_file(): # If exegol is clone from github, build context is accessible from root src return local_src else: @@ -51,13 +52,15 @@ def findBuildContextPath(cls) -> Path: possible_locations.append(Path(loc).parent.parent.parent) # Find a good match for test in possible_locations: - context_path = test / dockerbuild_folder_name + context_path = test / resource_folder if context_path.is_dir(): return context_path # Detect a venv context - return Path(site.PREFIXES[0]) / dockerbuild_folder_name + return Path(site.PREFIXES[0]) / resource_folder # Dynamically built attribute must be set after class initialization -ConstantConfig.build_context_path_obj = ConstantConfig.findBuildContextPath() +ConstantConfig.build_context_path_obj = ConstantConfig.findResourceContextPath("exegol-docker-build", "exegol-docker-build") ConstantConfig.build_context_path = str(ConstantConfig.build_context_path_obj) + +ConstantConfig.entrypoint_context_path_obj = ConstantConfig.findResourceContextPath("exegol-entrypoint", "exegol/utils/entrypoint/entrypoint.sh") diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index d2cfccf6..e3fe407e 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -12,6 +12,7 @@ from exegol.model.ExegolImage import ExegolImage from exegol.model.SelectableInterface import SelectableInterface from exegol.config.EnvInfo import EnvInfo +from exegol.utils.entrypoint.EntrypointUtils import getEntrypointTarData from exegol.utils.ExeLog import logger, console @@ -248,6 +249,10 @@ def postCreateSetup(self): :return: """ self.__applyXhostACL() + # Update entrypoint script in the container + self.__container.put_archive("/.exegol", getEntrypointTarData()) + if self.__container.status.lower() == "created": + self.__container.start() def __applyXhostACL(self): """ diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 2417f3ca..63d40b3e 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -101,29 +101,36 @@ def createContainer(cls, model: ExegolContainerTemplate, temporary: bool = False entrypoint, command = model.config.getEntrypointCommand(model.image.getEntrypointConfig()) logger.debug(f"Entrypoint: {entrypoint}") logger.debug(f"Cmd: {command}") + # The 'create' function must be called to create a container without starting it + # in order to hot patch the entrypoint.sh with wrapper features (the container will be started after postCreateSetup) + docker_create_function = cls.__client.containers.create + docker_args = {"image": model.image.getDockerRef(), + "entrypoint": entrypoint, + "command": command, + "detach": True, + "name": model.container_name, + "hostname": model.hostname, + "extra_hosts": {model.hostname: '127.0.0.1'}, + "devices": model.config.getDevices(), + "environment": model.config.getEnvs(), + "labels": model.config.getLabels(), + "network_mode": model.config.getNetworkMode(), + "ports": model.config.getPorts(), + "privileged": model.config.getPrivileged(), + "cap_add": model.config.getCapabilities(), + "sysctls": model.config.getSysctls(), + "shm_size": model.config.shm_size, + "stdin_open": model.config.interactive, + "tty": model.config.tty, + "mounts": model.config.getVolumes(), + "working_dir": model.config.getWorkingDir()} + if temporary: + # Only the 'run' function support the "remove" parameter + docker_create_function = cls.__client.containers.run + docker_args["remove"] = temporary + docker_args["auto_remove"] = temporary try: - container = cls.__client.containers.run(model.image.getDockerRef(), - entrypoint=entrypoint, - command=command, - detach=True, - name=model.container_name, - hostname=model.hostname, - extra_hosts={model.hostname: '127.0.0.1'}, - devices=model.config.getDevices(), - environment=model.config.getEnvs(), - labels=model.config.getLabels(), - network_mode=model.config.getNetworkMode(), - ports=model.config.getPorts(), - privileged=model.config.getPrivileged(), - cap_add=model.config.getCapabilities(), - sysctls=model.config.getSysctls(), - shm_size=model.config.shm_size, - stdin_open=model.config.interactive, - tty=model.config.tty, - mounts=model.config.getVolumes(), - remove=temporary, - auto_remove=temporary, - working_dir=model.config.getWorkingDir()) + container = docker_create_function(**docker_args) except APIError as err: message = err.explanation.decode('utf-8').replace('[', '\\[') if type(err.explanation) is bytes else err.explanation message = message.replace('[', '\\[') diff --git a/exegol/utils/entrypoint/EntrypointUtils.py b/exegol/utils/entrypoint/EntrypointUtils.py new file mode 100644 index 00000000..53ae131c --- /dev/null +++ b/exegol/utils/entrypoint/EntrypointUtils.py @@ -0,0 +1,30 @@ +import io +import tarfile + +from exegol import ConstantConfig +from exegol.utils.ExeLog import logger + + +def getEntrypointTarData(): + """The purpose of this class is to generate and overwrite the entrypoint script of exegol containers + to integrate the latest features, whatever the version of the image.""" + + # Load entrypoint data + script_path = ConstantConfig.entrypoint_context_path_obj + logger.debug(f"Entrypoint path: {str(script_path)}") + if not script_path.is_file(): + logger.error("Unable to find the entrypoint script! Your Exegol installation is probably broken...") + return None + with open(script_path, 'rb') as f: + raw = f.read() + data = io.BytesIO(initial_bytes=raw) + + # Create tar file + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode='w|') as entry_tar: + # Import file to tar object + info = tarfile.TarInfo(name="entrypoint.sh") + info.size = len(raw) + info.mode = 0o500 + entry_tar.addfile(info, fileobj=data) + return stream.getvalue() diff --git a/exegol/utils/entrypoint/__init__.py b/exegol/utils/entrypoint/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/exegol/utils/entrypoint/entrypoint.sh b/exegol/utils/entrypoint/entrypoint.sh new file mode 100644 index 00000000..59b5b2a8 --- /dev/null +++ b/exegol/utils/entrypoint/entrypoint.sh @@ -0,0 +1,100 @@ +#!/bin/bash +trap shutdown SIGTERM + +# Function specific +function load_setups() { + # Load custom setups (supported setups, and user setup) + [ -d /var/log/exegol ] || mkdir -p /var/log/exegol + if [[ ! -f /.exegol/.setup.lock ]]; then + # Execute initial setup if lock file doesn't exist + echo >/.exegol/.setup.lock + /.exegol/load_supported_setups.sh &>>/var/log/exegol/load_setups.log && gzip /var/log/exegol/load_setups.log + fi +} + +function endless() { + # Start action / endless + # Entrypoint for the container, in order to have a process hanging, to keep the container alive + # Alternative to running bash/zsh/whatever as entrypoint, which is longer to start and to stop and to very clean + read -u 2 +} + +function shutdown() { + # SIGTERM received (the container is stopping). + # Shutting down the container. + # Sending SIGTERM to all interactive process for proper closing + # shellcheck disable=SC2046 + kill $(pgrep -f -- openvpn | grep -vE '^1$') 2>/dev/null + # shellcheck disable=SC2046 + kill $(pgrep -x -f -- zsh) 2>/dev/null + # shellcheck disable=SC2046 + kill $(pgrep -x -f -- -zsh) 2>/dev/null + # shellcheck disable=SC2046 + kill $(pgrep -x -f -- bash) 2>/dev/null + # shellcheck disable=SC2046 + kill $(pgrep -x -f -- -bash) 2>/dev/null + # Wait for every active process to exit (e.g: shell logging compression, VPN closing) + wait_list="$(pgrep -f "(.log|start.sh)" | grep -vE '^1$')" + for i in $wait_list; do + # Waiting for: $i PID process to exit + tail --pid="$i" -f /dev/null + done + exit 0 +} + +function resolv_docker_host() { + # On docker desktop host, resolving the host.docker.internal before starting a VPN connection for GUI applications + docker_ip=$(getent hosts host.docker.internal | head -n1 | awk '{ print $1 }') + if [ "$docker_ip" ]; then + # Add docker internal host resolution to the hosts file to preserve access to the X server + echo "$docker_ip host.docker.internal" >>/etc/hosts + fi +} + +# Managed features +function default() { + load_setups + endless +} + +function ovpn() { + load_setups + [[ "$DISPLAY" == *"host.docker.internal"* ]] && resolv_docker_host + # Starting openvpn as a job with '&' to be able to receive SIGTERM signal and close everything properly + # shellcheck disable=SC2086 + openvpn --log-append /var/log/exegol/vpn.log $2 & + endless +} + +function cmd() { + load_setups + # echo "Executing: ${*:2}" + "${@:2}" +} + +function compatibility() { + # Older versions of exegol wrapper launch the container with the 'bash' command + # This command is now interpreted by the custom entrypoint + echo "Your version of Exegol wrapper is not up-to-date!" | tee ~/banner.txt + # If the command is bash, redirect to endless. Otherwise execute the command as job to keep the shutdown procedure available + if [ "$*" != "bash" ]; then + echo "Executing command in backwards compatibility mode" | tee ~/banner.txt + echo "$1 -c '${*:3}'" + $1 -c "${*:3}" & + fi + endless +} + +# Default action is "default" +func_name="${1:-default}" + +# Older versions of exegol wrapper launch the container with the 'bash' command +# This command is now interpreted by the custom entrypoint. Redirect execution to the raw execution for backward compatibility. +# shellcheck disable=SC2068 +[ "$func_name" == "bash" ] || [ "$func_name" == "zsh" ] && compatibility $@ + +# Dynamic execution +$func_name "$@" || ( + echo "An error occurred executing the '$func_name' action. Your image version is probably out of date for this feature. Please update your image." | tee ~/banner.txt + exit 1 +) diff --git a/setup.py b/setup.py index 2ec07207..e0012926 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ long_description = (here / 'README.md').read_text(encoding='utf-8') # Additional non-code data used by Exegol to build local docker image from source +## exegol-docker-build Dockerfiles source_directory = "exegol-docker-build" data_files_dict = {source_directory: [f"{source_directory}/Dockerfile"] + [str(profile) for profile in pathlib.Path(source_directory).rglob('*.dockerfile')]} data_files = [] @@ -22,6 +23,9 @@ if data_files_dict.get(key) is None: data_files_dict[key] = [] data_files_dict[key].append(str(path)) +## exegol-entrypoint script +data_files_dict["exegol-entrypoint"] = ["exegol/utils/entrypoint/entrypoint.sh"] + # Dict to tuple for k, v in data_files_dict.items(): data_files.append((k, v)) From 343e4b185e5276e27ce7dec9f807989f2c231a4d Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 22 Jul 2023 21:56:24 +0200 Subject: [PATCH 002/109] Wrapper can now show container startup status update --- exegol/model/ContainerConfig.py | 3 +- exegol/model/ExegolContainer.py | 24 ++++++++--- exegol/utils/ContainerLogStream.py | 57 +++++++++++++++++++++++++++ exegol/utils/entrypoint/entrypoint.sh | 16 ++++++-- 4 files changed, 90 insertions(+), 10 deletions(-) create mode 100644 exegol/utils/ContainerLogStream.py diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 1164a584..6e17f661 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -697,7 +697,8 @@ def getWorkingDir(self) -> str: def getEntrypointCommand(self, image_entrypoint: Optional[Union[str, List[str]]]) -> Tuple[Optional[List[str]], Union[List[str], str]]: """Get container entrypoint/command arguments. - This method support legacy configuration.""" + This method support legacy configuration. + The default container_entrypoint is '/.exegol/entrypoint.sh' and the default container_command is ['default'].""" if image_entrypoint is None: # Legacy mode if self.__container_command_legacy is None: diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index e3fe407e..7d1a3145 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -1,19 +1,21 @@ import os import shutil +from datetime import datetime from typing import Optional, Dict, Sequence, Tuple from docker.errors import NotFound, ImageNotFound from docker.models.containers import Container +from exegol.config.EnvInfo import EnvInfo from exegol.console.ExegolPrompt import Confirm from exegol.console.cli.ParametersManager import ParametersManager from exegol.model.ContainerConfig import ContainerConfig from exegol.model.ExegolContainerTemplate import ExegolContainerTemplate from exegol.model.ExegolImage import ExegolImage from exegol.model.SelectableInterface import SelectableInterface -from exegol.config.EnvInfo import EnvInfo -from exegol.utils.entrypoint.EntrypointUtils import getEntrypointTarData +from exegol.utils.ContainerLogStream import ContainerLogStream from exegol.utils.ExeLog import logger, console +from exegol.utils.entrypoint.EntrypointUtils import getEntrypointTarData class ExegolContainer(ExegolContainerTemplate, SelectableInterface): @@ -103,8 +105,20 @@ def start(self): if not self.isRunning(): logger.info(f"Starting container {self.name}") self.preStartSetup() - with console.status(f"Waiting to start {self.name}", spinner_style="blue"): - self.__container.start() + self.__start_container() + + def __start_container(self): + with console.status(f"Waiting to start {self.name}", spinner_style="blue") as progress: + start_date = datetime.utcnow() + self.__container.start() + try: + for line in ContainerLogStream(self.__container, start_date=start_date, timeout=2): + logger.verbose(line) + if line == "READY": + break + progress.update(status=f"[blue]\[Startup][/blue] {line}") + except KeyboardInterrupt: + logger.warning("User skip startup status updates. Spawning a shell now.") def stop(self, timeout: int = 10): """Stop the docker container""" @@ -252,7 +266,7 @@ def postCreateSetup(self): # Update entrypoint script in the container self.__container.put_archive("/.exegol", getEntrypointTarData()) if self.__container.status.lower() == "created": - self.__container.start() + self.__start_container() def __applyXhostACL(self): """ diff --git a/exegol/utils/ContainerLogStream.py b/exegol/utils/ContainerLogStream.py new file mode 100644 index 00000000..fcafa21c --- /dev/null +++ b/exegol/utils/ContainerLogStream.py @@ -0,0 +1,57 @@ +import asyncio +import concurrent.futures +import threading +import time +from datetime import datetime, timedelta +from typing import Union, List, Any, Optional + +from docker.models.containers import Container +from docker.types import CancellableStream + +from exegol.utils.ExeLog import logger + + +class ContainerLogStream: + + def __init__(self, container: Container, start_date: Optional[datetime] = None, timeout: int = 5): + self.__container = container + self.__start_date: datetime = datetime.utcnow() if start_date is None else start_date + self.__until_date: Optional[datetime] = None + self.__data_stream = None + self.__line_buffer = b'' + + self.__enable_timeout = timeout > 0 + self.__timeout_date: datetime = self.__start_date + timedelta(seconds=timeout) + + def __iter__(self): + return self + + def __next__(self): + data = self.next_line() + if data is None: + raise StopIteration + return data + + def next_line(self) -> Optional[str]: + """Get the next line of the stream""" + if self.__until_date is None: + self.__until_date = datetime.utcnow() + while True: + if self.__data_stream is None: + # The 'follow' mode cannot be used because there is no timeout mechanism and will stuck the process forever + self.__data_stream = self.__container.logs(stream=True, follow=False, since=self.__start_date, until=self.__until_date) + for streamed_char in self.__data_stream: + if (streamed_char == b'\r' or streamed_char == b'\n') and len(self.__line_buffer) > 0: + line = self.__line_buffer.decode('utf-8').strip() + self.__line_buffer = b"" + return line + else: + self.__enable_timeout = False # disable timeout if the container is up-to-date and support console logging + self.__line_buffer += streamed_char + if self.__enable_timeout and self.__until_date >= self.__timeout_date: + logger.debug("Container log stream timed-out") + return None + self.__data_stream = None + self.__start_date = self.__until_date + time.sleep(0.5) + self.__until_date = datetime.utcnow() diff --git a/exegol/utils/entrypoint/entrypoint.sh b/exegol/utils/entrypoint/entrypoint.sh index 59b5b2a8..33d3bd9b 100644 --- a/exegol/utils/entrypoint/entrypoint.sh +++ b/exegol/utils/entrypoint/entrypoint.sh @@ -1,4 +1,5 @@ #!/bin/bash +# SIGTERM received (the container is stopping, every process must be gracefully stopped before the timeout). trap shutdown SIGTERM # Function specific @@ -8,7 +9,10 @@ function load_setups() { if [[ ! -f /.exegol/.setup.lock ]]; then # Execute initial setup if lock file doesn't exist echo >/.exegol/.setup.lock - /.exegol/load_supported_setups.sh &>>/var/log/exegol/load_setups.log && gzip /var/log/exegol/load_setups.log + echo "Installing [green]my-resources[/green] custom setup ..." + # Run my-resources script. Logs starting with '[exegol]' will be print to the console and report back to the user through the wrapper. + /.exegol/load_supported_setups.sh |& tee /var/log/exegol/load_setups.log | grep -i '^\[exegol]' | sed "s/^\[exegol\]\s*//gi" + [ -f /var/log/exegol/load_setups.log ] && echo "Compressing [green]my-resources[/green] logs" && gzip /var/log/exegol/load_setups.log fi } @@ -16,13 +20,14 @@ function endless() { # Start action / endless # Entrypoint for the container, in order to have a process hanging, to keep the container alive # Alternative to running bash/zsh/whatever as entrypoint, which is longer to start and to stop and to very clean + echo "READY" read -u 2 } function shutdown() { - # SIGTERM received (the container is stopping). # Shutting down the container. # Sending SIGTERM to all interactive process for proper closing + pgrep guacd && /opt/tools/bin/desktop-stop # Stop webui desktop if started # shellcheck disable=SC2046 kill $(pgrep -f -- openvpn | grep -vE '^1$') 2>/dev/null # shellcheck disable=SC2046 @@ -33,8 +38,8 @@ function shutdown() { kill $(pgrep -x -f -- bash) 2>/dev/null # shellcheck disable=SC2046 kill $(pgrep -x -f -- -bash) 2>/dev/null - # Wait for every active process to exit (e.g: shell logging compression, VPN closing) - wait_list="$(pgrep -f "(.log|start.sh)" | grep -vE '^1$')" + # Wait for every active process to exit (e.g: shell logging compression, VPN closing, WebUI) + wait_list="$(pgrep -f "(.log|start.sh|tomcat)" | grep -vE '^1$')" for i in $wait_list; do # Waiting for: $i PID process to exit tail --pid="$i" -f /dev/null @@ -62,7 +67,9 @@ function ovpn() { [[ "$DISPLAY" == *"host.docker.internal"* ]] && resolv_docker_host # Starting openvpn as a job with '&' to be able to receive SIGTERM signal and close everything properly # shellcheck disable=SC2086 + echo "Starting [green]VPN[/green]" openvpn --log-append /var/log/exegol/vpn.log $2 & + sleep 2 # Waiting 2 seconds for the VPN to start before continuing endless } @@ -85,6 +92,7 @@ function compatibility() { endless } +echo "Starting exegol" # Default action is "default" func_name="${1:-default}" From 3349046c04f9c06d3fadbd31e5fe68cb910dad5f Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sun, 23 Jul 2023 17:58:21 +0200 Subject: [PATCH 003/109] Add dev comments --- exegol/model/ExegolContainer.py | 7 +++++++ exegol/utils/ContainerLogStream.py | 21 ++++++++++++--------- exegol/utils/entrypoint/entrypoint.sh | 13 ++++++++++--- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 7d1a3145..f020c407 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -108,16 +108,23 @@ def start(self): self.__start_container() def __start_container(self): + """ + This method start the container and display startup status update to the user. + :return: + """ with console.status(f"Waiting to start {self.name}", spinner_style="blue") as progress: start_date = datetime.utcnow() self.__container.start() try: + # Try to find log / startup messages. Will time out after 2 seconds if the image don't support status update through container logs. for line in ContainerLogStream(self.__container, start_date=start_date, timeout=2): logger.verbose(line) + # Once the last log "READY" is received, the startup sequence is over and the execution can continue if line == "READY": break progress.update(status=f"[blue]\[Startup][/blue] {line}") except KeyboardInterrupt: + # User can cancel startup logging with ctrl+C logger.warning("User skip startup status updates. Spawning a shell now.") def stop(self, timeout: int = 10): diff --git a/exegol/utils/ContainerLogStream.py b/exegol/utils/ContainerLogStream.py index fcafa21c..b4443563 100644 --- a/exegol/utils/ContainerLogStream.py +++ b/exegol/utils/ContainerLogStream.py @@ -14,12 +14,16 @@ class ContainerLogStream: def __init__(self, container: Container, start_date: Optional[datetime] = None, timeout: int = 5): + # Container to extract logs from self.__container = container + # Fetch more logs from this datetime self.__start_date: datetime = datetime.utcnow() if start_date is None else start_date self.__until_date: Optional[datetime] = None + # The data stream is returned from the docker SDK. It can contain multiple line at the same. self.__data_stream = None self.__line_buffer = b'' + # Enable timeout if > 0. Passed timeout_date, the iterator will stop. self.__enable_timeout = timeout > 0 self.__timeout_date: datetime = self.__start_date + timedelta(seconds=timeout) @@ -27,31 +31,30 @@ def __iter__(self): return self def __next__(self): - data = self.next_line() - if data is None: - raise StopIteration - return data - - def next_line(self) -> Optional[str]: """Get the next line of the stream""" if self.__until_date is None: self.__until_date = datetime.utcnow() while True: + # The data stream is fetch from the docker SDK once empty. if self.__data_stream is None: # The 'follow' mode cannot be used because there is no timeout mechanism and will stuck the process forever self.__data_stream = self.__container.logs(stream=True, follow=False, since=self.__start_date, until=self.__until_date) + # Parsed the data stream to extract characters and merge them into a line. for streamed_char in self.__data_stream: + # When detecting an end of line, the buffer is returned as a single line. if (streamed_char == b'\r' or streamed_char == b'\n') and len(self.__line_buffer) > 0: line = self.__line_buffer.decode('utf-8').strip() self.__line_buffer = b"" return line else: self.__enable_timeout = False # disable timeout if the container is up-to-date and support console logging - self.__line_buffer += streamed_char + self.__line_buffer += streamed_char # add characters to the line buffer + # When the data stream is empty, check if a timeout condition apply if self.__enable_timeout and self.__until_date >= self.__timeout_date: logger.debug("Container log stream timed-out") - return None + raise StopIteration + # Prepare the next iteration to fetch next logs self.__data_stream = None self.__start_date = self.__until_date - time.sleep(0.5) + time.sleep(0.5) # Wait for more logs self.__until_date = datetime.utcnow() diff --git a/exegol/utils/entrypoint/entrypoint.sh b/exegol/utils/entrypoint/entrypoint.sh index 33d3bd9b..2718b5c3 100644 --- a/exegol/utils/entrypoint/entrypoint.sh +++ b/exegol/utils/entrypoint/entrypoint.sh @@ -82,10 +82,10 @@ function cmd() { function compatibility() { # Older versions of exegol wrapper launch the container with the 'bash' command # This command is now interpreted by the custom entrypoint - echo "Your version of Exegol wrapper is not up-to-date!" | tee ~/banner.txt + echo "Your version of Exegol wrapper is not up-to-date!" | tee -a ~/banner.txt # If the command is bash, redirect to endless. Otherwise execute the command as job to keep the shutdown procedure available if [ "$*" != "bash" ]; then - echo "Executing command in backwards compatibility mode" | tee ~/banner.txt + echo "Executing command in backwards compatibility mode" | tee -a ~/banner.txt echo "$1 -c '${*:3}'" $1 -c "${*:3}" & fi @@ -101,8 +101,15 @@ func_name="${1:-default}" # shellcheck disable=SC2068 [ "$func_name" == "bash" ] || [ "$func_name" == "zsh" ] && compatibility $@ +### How "echo" works here with exegol ### +# Every message printed here will be displayed to the console logs of the container +# The container logs will be displayed by the wrapper to the user at startup through a progress animation (and a verbose line if -v is set) +# The logs written to ~/banner.txt will be printed to the user through the .zshrc file on each new session (until the file is removed). +# Using 'tee -a' after a command will save the output to a file AND to the console logs. +########################################## + # Dynamic execution $func_name "$@" || ( - echo "An error occurred executing the '$func_name' action. Your image version is probably out of date for this feature. Please update your image." | tee ~/banner.txt + echo "An error occurred executing the '$func_name' action. Your image version is probably out of date for this feature. Please update your image." | tee -a ~/banner.txt exit 1 ) From d32e30dc8cf5da048668dcef73ed72300198346f Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sun, 23 Jul 2023 19:18:12 +0200 Subject: [PATCH 004/109] Add tips for startup skip --- exegol/utils/ContainerLogStream.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/exegol/utils/ContainerLogStream.py b/exegol/utils/ContainerLogStream.py index b4443563..1f1b4f29 100644 --- a/exegol/utils/ContainerLogStream.py +++ b/exegol/utils/ContainerLogStream.py @@ -18,6 +18,7 @@ def __init__(self, container: Container, start_date: Optional[datetime] = None, self.__container = container # Fetch more logs from this datetime self.__start_date: datetime = datetime.utcnow() if start_date is None else start_date + self.__since_date = self.__start_date self.__until_date: Optional[datetime] = None # The data stream is returned from the docker SDK. It can contain multiple line at the same. self.__data_stream = None @@ -25,7 +26,11 @@ def __init__(self, container: Container, start_date: Optional[datetime] = None, # Enable timeout if > 0. Passed timeout_date, the iterator will stop. self.__enable_timeout = timeout > 0 - self.__timeout_date: datetime = self.__start_date + timedelta(seconds=timeout) + self.__timeout_date: datetime = self.__since_date + timedelta(seconds=timeout) + + # Hint message flag + self.__tips_sent = False + self.__tips_timedelta = self.__start_date + timedelta(seconds=15) def __iter__(self): return self @@ -38,7 +43,7 @@ def __next__(self): # The data stream is fetch from the docker SDK once empty. if self.__data_stream is None: # The 'follow' mode cannot be used because there is no timeout mechanism and will stuck the process forever - self.__data_stream = self.__container.logs(stream=True, follow=False, since=self.__start_date, until=self.__until_date) + self.__data_stream = self.__container.logs(stream=True, follow=False, since=self.__since_date, until=self.__until_date) # Parsed the data stream to extract characters and merge them into a line. for streamed_char in self.__data_stream: # When detecting an end of line, the buffer is returned as a single line. @@ -53,8 +58,14 @@ def __next__(self): if self.__enable_timeout and self.__until_date >= self.__timeout_date: logger.debug("Container log stream timed-out") raise StopIteration + elif not self.__tips_sent and self.__until_date >= self.__tips_timedelta: + self.__tips_sent = True + logger.info("Your start-up sequence takes time, your my-resource setup configuration may be significant.") + logger.info("[orange3]\[Tips][/orange3] If you want to skip startup update, " + "you can use [green]CTRL+C[/green] and spawn a shell immediately. " + "[blue](Startup sequence will continue in background)[/blue]") # Prepare the next iteration to fetch next logs self.__data_stream = None - self.__start_date = self.__until_date + self.__since_date = self.__until_date time.sleep(0.5) # Wait for more logs self.__until_date = datetime.utcnow() From 83e5ec2c6c9e0f914f26008c7f0b1bb5157ea764 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 26 Jul 2023 00:23:56 +0200 Subject: [PATCH 005/109] Add random password generation --- exegol/console/TUI.py | 27 ++++++++-- exegol/model/ContainerConfig.py | 87 +++++++++++++++++++++++++-------- exegol/model/ExegolContainer.py | 10 ++++ 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/exegol/console/TUI.py b/exegol/console/TUI.py index 1d70bb51..1544ca1e 100644 --- a/exegol/console/TUI.py +++ b/exegol/console/TUI.py @@ -396,8 +396,27 @@ def selectFromList(cls, @classmethod def printContainerRecap(cls, container: ExegolContainerTemplate): + """ + Build and print a rich table with every configuration of the container + :param container: Exegol container to print the table of + :return: + """ # Load the image status if it is not already set. container.image.autoLoad() + + recap = cls.__buildContainerRecapTable(container) + + logger.empty_line() + console.print(recap) + logger.empty_line() + + @staticmethod + def __buildContainerRecapTable(container: ExegolContainerTemplate): + """ + Build a rich table to recap in detail the configuration of a specified ExegolContainerTemplate or ExegolContainer + :param container: The container to fetch config from + :return: A rich table fully built + """ # Fetch data devices = container.config.getTextDevices(logger.isEnabledFor(ExeLog.VERBOSE)) envs = container.config.getTextEnvs(logger.isEnabledFor(ExeLog.VERBOSE)) @@ -407,12 +426,13 @@ def printContainerRecap(cls, container: ExegolContainerTemplate): volumes = container.config.getTextMounts(logger.isEnabledFor(ExeLog.VERBOSE)) creation_date = container.config.getTextCreationDate() comment = container.config.getComment() + passwd = container.config.getPasswd() # Color code privilege_color = "bright_magenta" path_color = "magenta" - logger.empty_line() + # Build table recap = Table(border_style="grey35", box=box.SQUARE, title_justify="left", show_header=True) recap.title = "[not italic]:white_medium_star: [/not italic][gold3][g]Container summary[/g][/gold3]" # Header @@ -427,6 +447,8 @@ def printContainerRecap(cls, container: ExegolContainerTemplate): container_info_header += f" [{color}]({container.image.getArch()})[/{color}]" recap.add_column(container_info_header) # Main features + if passwd: + recap.add_row(f"[bold blue]Credentials[/bold blue]", f"[deep_sky_blue3]{container.config.getUsername()}[/deep_sky_blue3] : [deep_sky_blue3]{passwd}[/deep_sky_blue3]") if comment: recap.add_row("[bold blue]Comment[/bold blue]", comment) if creation_date: @@ -466,8 +488,7 @@ def printContainerRecap(cls, container: ExegolContainerTemplate): recap.add_row("[bold blue]Systctls[/bold blue]", os.linesep.join( [f"[{privilege_color}]{key}[/{privilege_color}] = {getColor(value)[0]}{value}{getColor(value)[1]}" for key, value in sysctls.items()])) - console.print(recap) - logger.empty_line() + return recap @classmethod def __isInteractionAllowed(cls): diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 6e17f661..a859e240 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -1,6 +1,8 @@ import logging import os +import random import re +import string from datetime import datetime from pathlib import Path, PurePath from typing import Optional, List, Dict, Union, Tuple, cast @@ -35,9 +37,10 @@ class ContainerConfig: # Label features (wrapper method to enable the feature / label name) __label_features = {"enableShellLogging": "org.exegol.feature.shell_logging"} - # Label metadata (label name / [wrapper attribute to set the value, getter method to update labels]) - __label_metadata = {"org.exegol.metadata.creation_date": ["creation_date", "getCreationDate"], - "org.exegol.metadata.comment": ["comment", "getComment"]} + # Label metadata (label name / [setter method to set the value, getter method to update labels]) + __label_metadata = {"org.exegol.metadata.creation_date": ["setCreationDate", "getCreationDate"], + "org.exegol.metadata.comment": ["setComment", "getComment"], + "org.exegol.metadata.passwd": ["setPasswd", "getPasswd"]} def __init__(self, container: Optional[Container] = None): """Container config default value""" @@ -68,14 +71,18 @@ def __init__(self, container: Optional[Container] = None): self.__shell_logging: bool = False self.__start_delegate_mode: bool = False # Metadata attributes - self.creation_date: Optional[str] = None - self.comment: Optional[str] = None + self.__creation_date: Optional[str] = None + self.__comment: Optional[str] = None + self.__username: str = "root" + self.__passwd: Optional[str] = self.generateRandomPassword() if container is not None: self.__parseContainerConfig(container) def __parseContainerConfig(self, container: Container): """Parse Docker object to setup self configuration""" + # Reset default attributes + self.__passwd = None # Container Config section container_config = container.attrs.get("Config", {}) self.tty = container_config.get("Tty", True) @@ -130,10 +137,10 @@ def __parseLabels(self, labels: Dict[str, str]): logger.debug(f"Parsing label : {key}") if key.startswith("org.exegol.metadata."): # Find corresponding feature and attributes - for label, refs in self.__label_metadata.items(): + for label, refs in self.__label_metadata.items(): # Setter if label == key: - # reflective set of the metadata attribute (set metadata value to the corresponding attribute) - setattr(self, refs[0], value) + # reflective execution of setter method (set metadata value to the corresponding attribute) + getattr(self, refs[0])(value) break elif key.startswith("org.exegol.feature."): # Find corresponding feature and attributes @@ -421,9 +428,9 @@ def enableShellLogging(self): def addComment(self, comment): """Procedure to add comment to a container""" - if not self.comment: + if not self.__comment: logger.verbose("Config: Adding comment to container info") - self.comment = comment + self.__comment = comment self.addLabel("org.exegol.metadata.comment", comment) def __disableShellLogging(self): @@ -989,18 +996,61 @@ def removeLabel(self, key: str) -> bool: def getLabels(self) -> Dict[str, str]: """Labels config getter""" # Update metadata (from getter method) to the labels (on container creation) - for label_name, refs in self.__label_metadata.items(): + for label_name, refs in self.__label_metadata.items(): # Getter data = getattr(self, refs[1])() if data is not None: self.addLabel(label_name, data) return self.__labels + # Metadata labels getter / setter section + + def setCreationDate(self, creation_date: str): + """Set the container creation date parsed from the labels of an existing container.""" + self.__creation_date = creation_date + def getCreationDate(self) -> str: """Get container creation date. If the creation has not been set before, init as right now.""" - if self.creation_date is None: - self.creation_date = datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ') - return self.creation_date + if self.__creation_date is None: + self.__creation_date = datetime.now().strftime('%Y-%m-%dT%H:%M:%SZ') + return self.__creation_date + + def setComment(self, comment: str): + """Set the container comment parsed from the labels of an existing container.""" + self.__comment = comment + + def getComment(self) -> Optional[str]: + """Get the container comment. + If no comment has been supplied, returns None.""" + return self.__comment + + def setPasswd(self, passwd: str): + """ + Set the container root password parsed from the labels of an existing container. + This secret data can be stored inside labels because it is accessible only from the docker socket + which give direct access to the container anyway without password. + """ + self.__passwd = passwd + + def getPasswd(self) -> Optional[str]: + """ + Get the container password. + """ + return self.__passwd + + def getUsername(self) -> str: + """ + Get the container username. + """ + return self.__username + + @staticmethod + def generateRandomPassword(length: int = 30) -> str: + """ + Generate a new random password. + """ + charset = string.ascii_letters + string.digits + string.punctuation.replace("'", "") + return''.join(random.choice(charset) for i in range(length)) def getVpnName(self): """Get VPN Config name""" @@ -1076,14 +1126,9 @@ def getTextFeatures(self, verbose: bool = False) -> str: def getTextCreationDate(self) -> str: """Get the container creation date. If the creation date has not been supplied on the container, return empty string.""" - if self.creation_date is None: + if self.__creation_date is None: return "" - return datetime.strptime(self.creation_date, "%Y-%m-%dT%H:%M:%SZ").strftime("%d/%m/%Y %H:%M") - - def getComment(self) -> Optional[str]: - """Get the container comment. - If no comment has been supplied, returns None.""" - return self.comment + return datetime.strptime(self.__creation_date, "%Y-%m-%dT%H:%M:%SZ").strftime("%d/%m/%Y %H:%M") def getTextMounts(self, verbose: bool = False) -> str: """Text formatter for Mounts configurations. The verbose mode does not exclude technical volumes.""" diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index f020c407..d36e8dc5 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -274,6 +274,7 @@ def postCreateSetup(self): self.__container.put_archive("/.exegol", getEntrypointTarData()) if self.__container.status.lower() == "created": self.__start_container() + self.__updatePasswd() def __applyXhostACL(self): """ @@ -297,3 +298,12 @@ def __applyXhostACL(self): logger.debug(f"Adding xhost ACL to local:{self.hostname}") # add linux local ACL os.system(f"xhost +local:{self.hostname} > /dev/null") + + def __updatePasswd(self): + """ + If configured, update the password of the user inside the container. + :return: + """ + if self.config.getPasswd() is not None: + logger.debug(f"Updating the {self.config.getUsername()} password inside the container") + self.exec(["echo", f"'{self.config.getUsername()}:{self.config.getPasswd()}'", "|", "chpasswd"], quiet=True) From 8ac14728d3ab32d1bb9c1e0e81fe32eaebce1b3b Mon Sep 17 00:00:00 2001 From: Dramelac Date: Mon, 31 Jul 2023 20:45:40 +0200 Subject: [PATCH 006/109] Handling temporary container with entrypoint v2 --- exegol/manager/ExegolManager.py | 5 ++++- exegol/model/ContainerConfig.py | 4 ++-- exegol/model/ExegolContainer.py | 14 ++++++++------ 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index 82a03485..5aca626c 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -540,8 +540,11 @@ def __createTmpContainer(cls, image_name: Optional[str] = None) -> ExegolContain image: ExegolImage = cast(ExegolImage, cls.__loadOrInstallImage(override_image=image_name)) model = ExegolContainerTemplate(name, config, image, hostname=ParametersManager().hostname) + # Mount entrypoint as a volume (because in tmp mode the container is created with run instead of create method) + model.config.addVolume(str(ConstantConfig.entrypoint_context_path_obj), "/.exegol/entrypoint.sh", must_exist=True, read_only=True) + container = DockerUtils.createContainer(model, temporary=True) - container.postCreateSetup() + container.postCreateSetup(is_temporary=True) return container @classmethod diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index a859e240..9543ce23 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -28,8 +28,8 @@ class ContainerConfig: # Default hardcoded value __default_entrypoint_legacy = "bash" - __default_entrypoint = ["/.exegol/entrypoint.sh"] - __default_cmd = ["default"] + __default_entrypoint = ["/bin/bash", "/.exegol/entrypoint.sh"] + __default_cmd = [""] __default_shm_size = "64M" # Reference static config data diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index d36e8dc5..873b6211 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -264,17 +264,19 @@ def preStartSetup(self): """ self.__applyXhostACL() - def postCreateSetup(self): + def postCreateSetup(self, is_temporary: bool = False): """ Operation to be performed after creating a container :return: """ self.__applyXhostACL() - # Update entrypoint script in the container - self.__container.put_archive("/.exegol", getEntrypointTarData()) - if self.__container.status.lower() == "created": - self.__start_container() - self.__updatePasswd() + # if not a temporary container, apply custom config + if not is_temporary: + # Update entrypoint script in the container + self.__container.put_archive("/.exegol", getEntrypointTarData()) + if self.__container.status.lower() == "created": + self.__start_container() + self.__updatePasswd() def __applyXhostACL(self): """ From 73df0679b139354bc3310c0adc3fd88968c6aa43 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 3 Aug 2023 12:45:22 +0200 Subject: [PATCH 007/109] Update entrypoint v2 parameters --- exegol/manager/ExegolManager.py | 3 +- exegol/model/ContainerConfig.py | 58 ++++++--------- exegol/model/ExegolContainer.py | 15 +++- exegol/utils/DockerUtils.py | 2 +- exegol/utils/entrypoint/EntrypointUtils.py | 2 +- exegol/utils/entrypoint/entrypoint.sh | 86 +++++++++------------- 6 files changed, 74 insertions(+), 92 deletions(-) diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index 5aca626c..6c45f907 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -530,8 +530,7 @@ def __createTmpContainer(cls, image_name: Optional[str] = None) -> ExegolContain if ParametersManager().daemon: # Using formatShellCommand to support zsh aliases exec_payload, str_cmd = ExegolContainer.formatShellCommand(ParametersManager().exec, entrypoint_mode=True) - config.setLegacyContainerCommand(f"zsh -c '{exec_payload}'") - config.setContainerCommand("cmd", "zsh", "-c", exec_payload) + config.entrypointRunCmd() config.addEnv("CMD", str_cmd) config.addEnv("DISABLE_AUTO_UPDATE", "true") # Workspace must be disabled for temporary container because host directory is never deleted diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 9543ce23..fe1b58d2 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -29,7 +29,6 @@ class ContainerConfig: # Default hardcoded value __default_entrypoint_legacy = "bash" __default_entrypoint = ["/bin/bash", "/.exegol/entrypoint.sh"] - __default_cmd = [""] __default_shm_size = "64M" # Reference static config data @@ -64,12 +63,14 @@ def __init__(self, container: Optional[Container] = None): self.__workspace_custom_path: Optional[str] = None self.__workspace_dedicated_path: Optional[str] = None self.__disable_workspace: bool = False - self.__container_command_legacy: Optional[str] = None - self.__container_command: List[str] = self.__default_cmd self.__container_entrypoint: List[str] = self.__default_entrypoint self.__vpn_path: Optional[Union[Path, PurePath]] = None self.__shell_logging: bool = False self.__start_delegate_mode: bool = False + # Entrypoint features + self.__vpn_parameters: Optional[str] = None + self.__run_cmd: bool = False + self.__endless_container: bool = True # Metadata attributes self.__creation_date: Optional[str] = None self.__comment: Optional[str] = None @@ -480,12 +481,7 @@ def enableVPN(self, config_path: Optional[str] = None): # Add tun device, this device is needed to create VPN tunnels self.__addDevice("/dev/net/tun", mknod=True) # Sharing VPN configuration with the container - ovpn_parameters = self.__prepareVpnVolumes(config_path) - # Execution of the VPN daemon at container startup - if ovpn_parameters is not None: - vpn_cmd_legacy = f"bash -c 'mkdir -p /var/log/exegol; openvpn --log-append /var/log/exegol/vpn.log {ovpn_parameters}; bash'" - self.setLegacyContainerCommand(vpn_cmd_legacy) - self.setContainerCommand("ovpn", ovpn_parameters) + self.__vpn_parameters = self.__prepareVpnVolumes(config_path) def __prepareVpnVolumes(self, config_path: Optional[str]) -> Optional[str]: """Volumes must be prepared to share OpenVPN configuration files with the container. @@ -566,6 +562,7 @@ def __disableVPN(self) -> bool: if self.__vpn_path: logger.verbose('Removing VPN configuration') self.__vpn_path = None + self.__vpn_parameters = None self.__removeCapability("NET_ADMIN") self.__removeSysctl("net.ipv6.conf.all.disable_ipv6") self.removeDevice("/dev/net/tun") @@ -573,7 +570,6 @@ def __disableVPN(self) -> bool: self.removeVolume(container_path="/.exegol/vpn/auth/creds.txt") self.removeVolume(container_path="/.exegol/vpn/config/client.ovpn") self.removeVolume(container_path="/.exegol/vpn/config") - self.__restoreEntrypoint() return True return False @@ -622,21 +618,11 @@ def setNetworkMode(self, host_mode: Optional[bool]): host_mode = False self.__network_host = host_mode - def setContainerCommand(self, entrypoint_function: str, *parameters: str): - """Set the entrypoint command of the container. This command is executed at each startup. - This parameter is applied to the container at creation.""" - self.__container_command = [entrypoint_function] + list(parameters) - - def setLegacyContainerCommand(self, cmd: str): - """Set the entrypoint command of the container. This command is executed at each startup. - This parameter is applied to the container at creation. - This method is legacy, before the entrypoint exist (support images before 3.x.x).""" - self.__container_command_legacy = cmd - - def __restoreEntrypoint(self): - """Restore container's entrypoint to its default configuration""" - self.__container_command_legacy = None - self.__container_command = self.__default_cmd + def entrypointRunCmd(self, endless_mode=False): + """Enable the run_cmd feature of the entrypoint. This feature execute the command stored in the $CMD container environment variables. + The endless_mode parameter can specify if the container must stay alive after command execution or not""" + self.__run_cmd = True + self.__endless_container = endless_mode def addCapability(self, cap_string: str): """Add a linux capability to the container""" @@ -702,17 +688,21 @@ def getWorkingDir(self) -> str: """Get default container's default working directory path""" return "/" if self.__disable_workspace else "/workspace" - def getEntrypointCommand(self, image_entrypoint: Optional[Union[str, List[str]]]) -> Tuple[Optional[List[str]], Union[List[str], str]]: + def getEntrypointCommand(self) -> Tuple[Optional[List[str]], Union[List[str], str]]: """Get container entrypoint/command arguments. - This method support legacy configuration. - The default container_entrypoint is '/.exegol/entrypoint.sh' and the default container_command is ['default'].""" - if image_entrypoint is None: - # Legacy mode - if self.__container_command_legacy is None: - return [self.__default_entrypoint_legacy], [] - return None, self.__container_command_legacy + The default container_entrypoint is '/bin/bash /.exegol/entrypoint.sh' and the default container_command is ['load_setups', 'endless'].""" + entrypoint_actions = [] + if self.__my_resources: + entrypoint_actions.append("load_setups") + if self.__vpn_path is not None: + entrypoint_actions.append(f"ovpn {self.__vpn_parameters}") + if self.__run_cmd: + entrypoint_actions.append("run_cmd") + if self.__endless_container: + entrypoint_actions.append("endless") else: - return self.__container_entrypoint, self.__container_command + entrypoint_actions.append("end") + return self.__container_entrypoint, entrypoint_actions def getShellCommand(self) -> str: """Get container command for opening a new shell""" diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 873b6211..302b8751 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -118,10 +118,17 @@ def __start_container(self): try: # Try to find log / startup messages. Will time out after 2 seconds if the image don't support status update through container logs. for line in ContainerLogStream(self.__container, start_date=start_date, timeout=2): - logger.verbose(line) # Once the last log "READY" is received, the startup sequence is over and the execution can continue if line == "READY": break + elif line.startswith('[W]'): + line = line.replace('[W]', '') + logger.warning(line) + elif line.startswith('[E]'): + line = line.replace('[E]', '') + logger.error(line) + else: + logger.verbose(line) progress.update(status=f"[blue]\[Startup][/blue] {line}") except KeyboardInterrupt: # User can cancel startup logging with ctrl+C @@ -191,11 +198,11 @@ def formatShellCommand(command: Sequence[str], quiet: bool = False, entrypoint_m - The first return argument is the payload to execute with every pre-routine for zsh. - The second return argument is the command itself in str format.""" # Using base64 to escape special characters - str_cmd = ' '.join(command) + str_cmd = ' '.join(command).replace('"', '\\"') if not quiet: logger.success(f"Command received: {str_cmd}") # ZSH pre-routine: Load zsh aliases and call eval to force aliases interpretation - cmd = f'autoload -Uz compinit; compinit; source ~/.zshrc; eval $CMD' + cmd = f'autoload -Uz compinit; compinit; source ~/.zshrc; eval "$CMD"' if not entrypoint_mode: # For direct execution, the full command must be supplied not just the zsh argument cmd = f"zsh -c '{cmd}'" @@ -273,7 +280,7 @@ def postCreateSetup(self, is_temporary: bool = False): # if not a temporary container, apply custom config if not is_temporary: # Update entrypoint script in the container - self.__container.put_archive("/.exegol", getEntrypointTarData()) + self.__container.put_archive("/", getEntrypointTarData()) if self.__container.status.lower() == "created": self.__start_container() self.__updatePasswd() diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 63d40b3e..ef4de21c 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -98,7 +98,7 @@ def createContainer(cls, model: ExegolContainerTemplate, temporary: bool = False docker_volume = cls.__loadDockerVolume(volume_path=volume['Source'], volume_name=volume['Target']) if docker_volume is None: logger.warning(f"Error while creating docker volume '{volume['Target']}'") - entrypoint, command = model.config.getEntrypointCommand(model.image.getEntrypointConfig()) + entrypoint, command = model.config.getEntrypointCommand() logger.debug(f"Entrypoint: {entrypoint}") logger.debug(f"Cmd: {command}") # The 'create' function must be called to create a container without starting it diff --git a/exegol/utils/entrypoint/EntrypointUtils.py b/exegol/utils/entrypoint/EntrypointUtils.py index 53ae131c..c3b29241 100644 --- a/exegol/utils/entrypoint/EntrypointUtils.py +++ b/exegol/utils/entrypoint/EntrypointUtils.py @@ -23,7 +23,7 @@ def getEntrypointTarData(): stream = io.BytesIO() with tarfile.open(fileobj=stream, mode='w|') as entry_tar: # Import file to tar object - info = tarfile.TarInfo(name="entrypoint.sh") + info = tarfile.TarInfo(name="/.exegol/entrypoint.sh") info.size = len(raw) info.mode = 0o500 entry_tar.addfile(info, fileobj=data) diff --git a/exegol/utils/entrypoint/entrypoint.sh b/exegol/utils/entrypoint/entrypoint.sh index 2718b5c3..7946292a 100644 --- a/exegol/utils/entrypoint/entrypoint.sh +++ b/exegol/utils/entrypoint/entrypoint.sh @@ -9,19 +9,28 @@ function load_setups() { if [[ ! -f /.exegol/.setup.lock ]]; then # Execute initial setup if lock file doesn't exist echo >/.exegol/.setup.lock - echo "Installing [green]my-resources[/green] custom setup ..." # Run my-resources script. Logs starting with '[exegol]' will be print to the console and report back to the user through the wrapper. - /.exegol/load_supported_setups.sh |& tee /var/log/exegol/load_setups.log | grep -i '^\[exegol]' | sed "s/^\[exegol\]\s*//gi" - [ -f /var/log/exegol/load_setups.log ] && echo "Compressing [green]my-resources[/green] logs" && gzip /var/log/exegol/load_setups.log + if [ -f /.exegol/load_supported_setupsd.sh ]; then + echo "Installing [green]my-resources[/green] custom setup ..." + /.exegol/load_supported_setups.sh |& tee /var/log/exegol/load_setups.log | grep -i '^\[exegol]' | sed "s/^\[exegol\]\s*//gi" + [ -f /var/log/exegol/load_setups.log ] && echo "Compressing [green]my-resources[/green] logs" && gzip /var/log/exegol/load_setups.log && echo "My-resources loaded" + else + echo "[W]Your exegol image doesn't support my-resources custom setup!" + fi fi } +function end() { + echo "READY" +} + function endless() { # Start action / endless + end # Entrypoint for the container, in order to have a process hanging, to keep the container alive # Alternative to running bash/zsh/whatever as entrypoint, which is longer to start and to stop and to very clean - echo "READY" - read -u 2 + # shellcheck disable=SC2162 + read -u 2 # read from stderr => endlessly wait effortlessly } function shutdown() { @@ -47,7 +56,7 @@ function shutdown() { exit 0 } -function resolv_docker_host() { +function _resolv_docker_host() { # On docker desktop host, resolving the host.docker.internal before starting a VPN connection for GUI applications docker_ip=$(getent hosts host.docker.internal | head -n1 | awk '{ print $1 }') if [ "$docker_ip" ]; then @@ -56,60 +65,37 @@ function resolv_docker_host() { fi } -# Managed features -function default() { - load_setups - endless -} - function ovpn() { - load_setups - [[ "$DISPLAY" == *"host.docker.internal"* ]] && resolv_docker_host + [[ "$DISPLAY" == *"host.docker.internal"* ]] && _resolv_docker_host # Starting openvpn as a job with '&' to be able to receive SIGTERM signal and close everything properly - # shellcheck disable=SC2086 echo "Starting [green]VPN[/green]" - openvpn --log-append /var/log/exegol/vpn.log $2 & + openvpn --log-append /var/log/exegol/vpn.log "$@" & sleep 2 # Waiting 2 seconds for the VPN to start before continuing - endless -} - -function cmd() { - load_setups - # echo "Executing: ${*:2}" - "${@:2}" } -function compatibility() { - # Older versions of exegol wrapper launch the container with the 'bash' command - # This command is now interpreted by the custom entrypoint - echo "Your version of Exegol wrapper is not up-to-date!" | tee -a ~/banner.txt - # If the command is bash, redirect to endless. Otherwise execute the command as job to keep the shutdown procedure available - if [ "$*" != "bash" ]; then - echo "Executing command in backwards compatibility mode" | tee -a ~/banner.txt - echo "$1 -c '${*:3}'" - $1 -c "${*:3}" & - fi - endless +function run_cmd() { + /bin/zsh -c "autoload -Uz compinit; compinit; source ~/.zshrc; eval \"$CMD\"" } -echo "Starting exegol" -# Default action is "default" -func_name="${1:-default}" - -# Older versions of exegol wrapper launch the container with the 'bash' command -# This command is now interpreted by the custom entrypoint. Redirect execution to the raw execution for backward compatibility. -# shellcheck disable=SC2068 -[ "$func_name" == "bash" ] || [ "$func_name" == "zsh" ] && compatibility $@ - -### How "echo" works here with exegol ### +##### How "echo" works here with exegol ##### +# # Every message printed here will be displayed to the console logs of the container # The container logs will be displayed by the wrapper to the user at startup through a progress animation (and a verbose line if -v is set) # The logs written to ~/banner.txt will be printed to the user through the .zshrc file on each new session (until the file is removed). # Using 'tee -a' after a command will save the output to a file AND to the console logs. -########################################## +# +############################################# +echo "Starting exegol" + +### Argument parsing -# Dynamic execution -$func_name "$@" || ( - echo "An error occurred executing the '$func_name' action. Your image version is probably out of date for this feature. Please update your image." | tee -a ~/banner.txt - exit 1 -) +# Par each parameter +for arg in "$@"; do + # Check if the function exist + function_name=$(echo "$arg" | cut -d ' ' -f 1) + if declare -f "$function_name" > /dev/null; then + $arg + else + echo "The function '$arg' doesn't exist." + fi +done From 395a08b508b58e9feb394020c4919e80d9a6c91a Mon Sep 17 00:00:00 2001 From: Dramelac Date: Fri, 4 Aug 2023 16:44:06 +0200 Subject: [PATCH 008/109] Reorganize ContainerConfig with sections --- exegol/model/ContainerConfig.py | 374 +++++++++++---------- exegol/utils/ContainerLogStream.py | 1 + exegol/utils/entrypoint/EntrypointUtils.py | 2 +- 3 files changed, 195 insertions(+), 182 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index fe1b58d2..396c6f7c 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -11,16 +11,16 @@ from docker.types import Mount from rich.prompt import Prompt +from exegol.config.EnvInfo import EnvInfo +from exegol.config.UserConfig import UserConfig from exegol.console.ConsoleFormat import boolFormatter, getColor from exegol.console.ExegolPrompt import Confirm from exegol.console.cli.ParametersManager import ParametersManager from exegol.exceptions.ExegolExceptions import ProtocolNotSupported, CancelOperation from exegol.model.ExegolModules import ExegolModules from exegol.utils import FsUtils -from exegol.config.EnvInfo import EnvInfo from exegol.utils.ExeLog import logger, ExeLog from exegol.utils.GuiUtils import GuiUtils -from exegol.config.UserConfig import UserConfig class ContainerConfig: @@ -80,6 +80,8 @@ def __init__(self, container: Optional[Container] = None): if container is not None: self.__parseContainerConfig(container) + # ===== Config parsing section ===== + def __parseContainerConfig(self, container: Container): """Parse Docker object to setup self configuration""" # Reset default attributes @@ -204,6 +206,8 @@ def __parseMounts(self, mounts: Optional[List[Dict]], name: str): self.__vpn_path = obj_path logger.debug(f"Loading VPN config: {self.__vpn_path.name}") + # ===== Feature section ===== + def interactiveConfig(self, container_name: str) -> List[str]: """Interactive procedure allowing the user to configure its new container""" logger.info("Starting interactive configuration") @@ -373,13 +377,6 @@ def __disableSharedTimezone(self): self.removeVolume("/etc/timezone") self.removeVolume("/etc/localtime") - def setPrivileged(self, status: bool = True): - """Set container as privileged""" - logger.verbose(f"Config: Setting container privileged as {status}") - if status: - logger.warning("Setting container as privileged (this exposes the host to security risks)") - self.__privileged = status - def enableMyResources(self): """Procedure to enable shared volume feature""" # TODO test my resources cross shell source (WSL / PSH) on Windows @@ -427,13 +424,6 @@ def enableShellLogging(self): self.__shell_logging = True self.addLabel(self.__label_features.get('enableShellLogging', 'org.exegol.error'), "Enabled") - def addComment(self, comment): - """Procedure to add comment to a container""" - if not self.__comment: - logger.verbose("Config: Adding comment to container info") - self.__comment = comment - self.addLabel("org.exegol.metadata.comment", comment) - def __disableShellLogging(self): """Procedure to disable exegol shell logging feature""" if self.__shell_logging: @@ -446,17 +436,6 @@ def enableCwdShare(self): self.__workspace_custom_path = os.getcwd() logger.verbose(f"Config: Sharing current workspace directory {self.__workspace_custom_path}") - def setWorkspaceShare(self, host_directory): - """Procedure to share a specific directory with the /workspace of the container""" - path = Path(host_directory).expanduser().absolute() - try: - if not path.is_dir() and path.exists(): - logger.critical("The specified workspace is not a directory!") - except PermissionError as e: - logger.critical(f"Unable to use the supplied workspace directory: {e}") - logger.verbose(f"Config: Sharing workspace directory {path}") - self.__workspace_custom_path = str(path) - def enableVPN(self, config_path: Optional[str] = None): """Configure a VPN profile for container startup""" # Check host mode : custom (allows you to isolate the VPN connection from the host's network) @@ -483,6 +462,37 @@ def enableVPN(self, config_path: Optional[str] = None): # Sharing VPN configuration with the container self.__vpn_parameters = self.__prepareVpnVolumes(config_path) + def __disableVPN(self) -> bool: + """Remove a VPN profile for container startup (Only for interactive config)""" + if self.__vpn_path: + logger.verbose('Removing VPN configuration') + self.__vpn_path = None + self.__vpn_parameters = None + self.__removeCapability("NET_ADMIN") + self.__removeSysctl("net.ipv6.conf.all.disable_ipv6") + self.removeDevice("/dev/net/tun") + # Try to remove each possible volume + self.removeVolume(container_path="/.exegol/vpn/auth/creds.txt") + self.removeVolume(container_path="/.exegol/vpn/config/client.ovpn") + self.removeVolume(container_path="/.exegol/vpn/config") + return True + return False + + def disableDefaultWorkspace(self): + """Allows you to disable the default workspace volume""" + # If a custom workspace is not define, disable workspace + if self.__workspace_custom_path is None: + self.__disable_workspace = True + + def addComment(self, comment): + """Procedure to add comment to a container""" + if not self.__comment: + logger.verbose("Config: Adding comment to container info") + self.__comment = comment + self.addLabel("org.exegol.metadata.comment", comment) + + # ===== Functional / technical methods section ===== + def __prepareVpnVolumes(self, config_path: Optional[str]) -> Optional[str]: """Volumes must be prepared to share OpenVPN configuration files with the container. Depending on the user's settings, different configurations can be applied. @@ -557,28 +567,6 @@ def __checkVPNConfigDNS(vpn_path: Union[str, Path]): logger.info("Press enter to continue or Ctrl+C to cancel the operation") input() - def __disableVPN(self) -> bool: - """Remove a VPN profile for container startup (Only for interactive config)""" - if self.__vpn_path: - logger.verbose('Removing VPN configuration') - self.__vpn_path = None - self.__vpn_parameters = None - self.__removeCapability("NET_ADMIN") - self.__removeSysctl("net.ipv6.conf.all.disable_ipv6") - self.removeDevice("/dev/net/tun") - # Try to remove each possible volume - self.removeVolume(container_path="/.exegol/vpn/auth/creds.txt") - self.removeVolume(container_path="/.exegol/vpn/config/client.ovpn") - self.removeVolume(container_path="/.exegol/vpn/config") - return True - return False - - def disableDefaultWorkspace(self): - """Allows you to disable the default workspace volume""" - # If a custom workspace is not define, disable workspace - if self.__workspace_custom_path is None: - self.__disable_workspace = True - def prepareShare(self, share_name: str): """Add workspace share before container creation""" for mount in self.__mounts: @@ -604,6 +592,68 @@ def rollback_preparation(self, share_name: str): if directory_path.is_dir(): directory_path.rmdir() + def entrypointRunCmd(self, endless_mode=False): + """Enable the run_cmd feature of the entrypoint. This feature execute the command stored in the $CMD container environment variables. + The endless_mode parameter can specify if the container must stay alive after command execution or not""" + self.__run_cmd = True + self.__endless_container = endless_mode + + def getEntrypointCommand(self) -> Tuple[Optional[List[str]], Union[List[str], str]]: + """Get container entrypoint/command arguments. + The default container_entrypoint is '/bin/bash /.exegol/entrypoint.sh' and the default container_command is ['load_setups', 'endless'].""" + entrypoint_actions = [] + if self.__my_resources: + entrypoint_actions.append("load_setups") + if self.__vpn_path is not None: + entrypoint_actions.append(f"ovpn {self.__vpn_parameters}") + if self.__run_cmd: + entrypoint_actions.append("run_cmd") + if self.__endless_container: + entrypoint_actions.append("endless") + else: + entrypoint_actions.append("end") + return self.__container_entrypoint, entrypoint_actions + + def getShellCommand(self) -> str: + """Get container command for opening a new shell""" + # If shell logging was enabled at container creation, it'll always be enabled for every shell. + # If not, it can be activated per shell basis + if self.__shell_logging or ParametersManager().log: + if self.__start_delegate_mode: + # Use a start.sh script to handle the feature with the tools and feature corresponding to the image version + # Start shell_logging feature using the user's specified method with the configured default shell w/ or w/o compression at the end + return f"/.exegol/start.sh shell_logging {ParametersManager().log_method} {ParametersManager().shell} {UserConfig().shell_logging_compress ^ ParametersManager().log_compress}" + else: + # Legacy command support + if ParametersManager().log_method != "script": + logger.warning("Your image version does not allow customization of the shell logging method. Using legacy script method.") + compression_cmd = '' + if UserConfig().shell_logging_compress ^ ParametersManager().log_compress: + compression_cmd = 'echo "Compressing logs, please wait..."; gzip $filelog; ' + return f"bash -c 'umask 007; mkdir -p /workspace/logs/; filelog=/workspace/logs/$(date +%d-%m-%Y_%H-%M-%S)_shell.log; script -qefac {ParametersManager().shell} $filelog; {compression_cmd}exit'" + return ParametersManager().shell + + @staticmethod + def generateRandomPassword(length: int = 30) -> str: + """ + Generate a new random password. + """ + charset = string.ascii_letters + string.digits + string.punctuation.replace("'", "") + return ''.join(random.choice(charset) for i in range(length)) + + # ===== Apply config section ===== + + def setWorkspaceShare(self, host_directory): + """Procedure to share a specific directory with the /workspace of the container""" + path = Path(host_directory).expanduser().absolute() + try: + if not path.is_dir() and path.exists(): + logger.critical("The specified workspace is not a directory!") + except PermissionError as e: + logger.critical(f"Unable to use the supplied workspace directory: {e}") + logger.verbose(f"Config: Sharing workspace directory {path}") + self.__workspace_custom_path = str(path) + def setNetworkMode(self, host_mode: Optional[bool]): """Set container's network mode, true for host, false for bridge""" if host_mode is None: @@ -618,11 +668,12 @@ def setNetworkMode(self, host_mode: Optional[bool]): host_mode = False self.__network_host = host_mode - def entrypointRunCmd(self, endless_mode=False): - """Enable the run_cmd feature of the entrypoint. This feature execute the command stored in the $CMD container environment variables. - The endless_mode parameter can specify if the container must stay alive after command execution or not""" - self.__run_cmd = True - self.__endless_container = endless_mode + def setPrivileged(self, status: bool = True): + """Set container as privileged""" + logger.verbose(f"Config: Setting container privileged as {status}") + if status: + logger.warning("Setting container as privileged (this exposes the host to security risks)") + self.__privileged = status def addCapability(self, cap_string: str): """Add a linux capability to the container""" @@ -665,13 +716,6 @@ def getNetworkMode(self) -> str: """Network mode, docker term getter""" return "host" if self.__network_host else "bridge" - def getTextNetworkMode(self) -> str: - """Network mode, text getter""" - network_mode = "host" if self.__network_host else "bridge" - if self.__vpn_path: - network_mode += " with VPN" - return network_mode - def getPrivileged(self) -> bool: """Privileged getter""" return self.__privileged @@ -688,47 +732,12 @@ def getWorkingDir(self) -> str: """Get default container's default working directory path""" return "/" if self.__disable_workspace else "/workspace" - def getEntrypointCommand(self) -> Tuple[Optional[List[str]], Union[List[str], str]]: - """Get container entrypoint/command arguments. - The default container_entrypoint is '/bin/bash /.exegol/entrypoint.sh' and the default container_command is ['load_setups', 'endless'].""" - entrypoint_actions = [] - if self.__my_resources: - entrypoint_actions.append("load_setups") - if self.__vpn_path is not None: - entrypoint_actions.append(f"ovpn {self.__vpn_parameters}") - if self.__run_cmd: - entrypoint_actions.append("run_cmd") - if self.__endless_container: - entrypoint_actions.append("endless") - else: - entrypoint_actions.append("end") - return self.__container_entrypoint, entrypoint_actions - - def getShellCommand(self) -> str: - """Get container command for opening a new shell""" - # If shell logging was enabled at container creation, it'll always be enabled for every shell. - # If not, it can be activated per shell basis - if self.__shell_logging or ParametersManager().log: - if self.__start_delegate_mode: - # Use a start.sh script to handle the feature with the tools and feature corresponding to the image version - # Start shell_logging feature using the user's specified method with the configured default shell w/ or w/o compression at the end - return f"/.exegol/start.sh shell_logging {ParametersManager().log_method} {ParametersManager().shell} {UserConfig().shell_logging_compress ^ ParametersManager().log_compress}" - else: - # Legacy command support - if ParametersManager().log_method != "script": - logger.warning("Your image version does not allow customization of the shell logging method. Using legacy script method.") - compression_cmd = '' - if UserConfig().shell_logging_compress ^ ParametersManager().log_compress: - compression_cmd = 'echo "Compressing logs, please wait..."; gzip $filelog; ' - return f"bash -c 'umask 007; mkdir -p /workspace/logs/; filelog=/workspace/logs/$(date +%d-%m-%Y_%H-%M-%S)_shell.log; script -qefac {ParametersManager().shell} $filelog; {compression_cmd}exit'" - return ParametersManager().shell - def getHostWorkspacePath(self) -> str: """Get private volume path (None if not set)""" if self.__workspace_custom_path: return FsUtils.resolvStrPath(self.__workspace_custom_path) elif self.__workspace_dedicated_path: - return FsUtils.resolvStrPath(self.__workspace_dedicated_path) + return self.getPrivateVolumePath() return "not found :(" def getPrivateVolumePath(self) -> str: @@ -827,34 +836,6 @@ def addVolume(self, mount = Mount(container_path, host_path, read_only=read_only, type=volume_type) self.__mounts.append(mount) - def addRawVolume(self, volume_string): - """Add a volume to the container configuration from raw text input. - Expected format is: /source/path:/target/mount:rw""" - logger.debug(f"Parsing raw volume config: {volume_string}") - parsing = re.match(r'^((\w:)?([\\/][\w .,:\-|()&;]*)+):(([\\/][\w .,\-|()&;]*)+)(:(ro|rw))?$', - volume_string) - if parsing: - host_path = parsing.group(1) - container_path = parsing.group(4) - mode = parsing.group(7) - if mode is None or mode == "rw": - readonly = False - elif mode == "ro": - readonly = True - else: - logger.error(f"Error on volume config, mode: {mode} not recognized.") - readonly = False - logger.debug( - f"Adding a volume from '{host_path}' to '{container_path}' as {'readonly' if readonly else 'read/write'}") - try: - self.addVolume(host_path, container_path, readonly) - except CancelOperation as e: - logger.error(f"The following volume couldn't be created [magenta]{volume_string}[/magenta]. {e}") - if not Confirm("Do you want to continue without this volume ?", False): - exit(0) - else: - logger.critical(f"Volume '{volume_string}' cannot be parsed. Exiting.") - def removeVolume(self, host_path: Optional[str] = None, container_path: Optional[str] = None) -> bool: """Remove a volume from the container configuration (Only before container creation)""" if host_path is None and container_path is None: @@ -891,14 +872,6 @@ def __addDevice(self, perm += 'm' self.__devices.append(f"{device_source}:{device_dest}:{perm}") - def addUserDevice(self, user_device_config: str): - """Add a device from a user parameters""" - if EnvInfo.isDockerDesktop(): - logger.warning("Docker desktop (Windows & macOS) does not support USB device passthrough.") - logger.verbose("Official doc: https://docs.docker.com/desktop/faqs/#can-i-pass-through-a-usb-device-to-a-container") - logger.critical("Device configuration cannot be applied, aborting operation.") - self.__addDevice(user_device_config) - def removeDevice(self, device_source: str) -> bool: """Remove a device from the container configuration (Only before container creation)""" for i in range(len(self.__devices)): @@ -926,29 +899,10 @@ def removeEnv(self, key: str) -> bool: # When the Key is not present in the dictionary return False - def addRawEnv(self, env: str): - """Parse and add an environment variable from raw user input""" - key, value = self.__parseUserEnv(env) - self.addEnv(key, value) - def getEnvs(self) -> Dict[str, str]: """Envs config getter""" return self.__envs - @classmethod - def __parseUserEnv(cls, env: str) -> Tuple[str, str]: - env_args = env.split('=') - key = env_args[0] - if len(env_args) < 2: - value = os.getenv(env, '') - if not value: - logger.critical(f"Incorrect env syntax ({env}). Please use this format: KEY=value") - else: - logger.success(f"Using system value for env {env}.") - else: - value = '='.join(env_args[1:]) - return key, value - def getShellEnvs(self) -> List[str]: """Overriding envs when opening a shell""" result = [] @@ -970,6 +924,24 @@ def getShellEnvs(self) -> List[str]: result.append(f"{key}={value}") return result + def addPort(self, + port_host: int, + port_container: Union[int, str], + protocol: str = 'tcp', + host_ip: str = '0.0.0.0'): + """Add port NAT config, only applicable on bridge network mode.""" + if self.__network_host: + logger.warning("Port sharing is configured, disabling the host network mode.") + self.setNetworkMode(False) + if protocol.lower() not in ['tcp', 'udp', 'sctp']: + raise ProtocolNotSupported(f"Unknown protocol '{protocol}'") + logger.debug(f"Adding port {host_ip}:{port_host} -> {port_container}/{protocol}") + self.__ports[f"{port_container}/{protocol}"] = (host_ip, port_host) + + def getPorts(self) -> Dict[str, Optional[Union[int, Tuple[str, int], List[int], List[Dict[str, Union[int, str]]]]]]: + """Ports config getter""" + return self.__ports + def addLabel(self, key: str, value: str): """Add a custom label to the container configuration""" self.__labels[key] = value @@ -992,7 +964,7 @@ def getLabels(self) -> Dict[str, str]: self.addLabel(label_name, data) return self.__labels - # Metadata labels getter / setter section + # ===== Metadata labels getter / setter section ===== def setCreationDate(self, creation_date: str): """Set the container creation date parsed from the labels of an existing container.""" @@ -1034,33 +1006,43 @@ def getUsername(self) -> str: """ return self.__username - @staticmethod - def generateRandomPassword(length: int = 30) -> str: - """ - Generate a new random password. - """ - charset = string.ascii_letters + string.digits + string.punctuation.replace("'", "") - return''.join(random.choice(charset) for i in range(length)) + # ===== User parameter parsing section ===== - def getVpnName(self): - """Get VPN Config name""" - if self.__vpn_path is None: - return "[bright_black]N/A[/bright_black] " - return f"[deep_sky_blue3]{self.__vpn_path.name}[/deep_sky_blue3]" + def addRawVolume(self, volume_string): + """Add a volume to the container configuration from raw text input. + Expected format is: /source/path:/target/mount:rw""" + logger.debug(f"Parsing raw volume config: {volume_string}") + parsing = re.match(r'^((\w:)?([\\/][\w .,:\-|()&;]*)+):(([\\/][\w .,\-|()&;]*)+)(:(ro|rw))?$', + volume_string) + if parsing: + host_path = parsing.group(1) + container_path = parsing.group(4) + mode = parsing.group(7) + if mode is None or mode == "rw": + readonly = False + elif mode == "ro": + readonly = True + else: + logger.error(f"Error on volume config, mode: {mode} not recognized.") + readonly = False + logger.debug( + f"Adding a volume from '{host_path}' to '{container_path}' as {'readonly' if readonly else 'read/write'}") + try: + self.addVolume(host_path, container_path, readonly) + except CancelOperation as e: + logger.error(f"The following volume couldn't be created [magenta]{volume_string}[/magenta]. {e}") + if not Confirm("Do you want to continue without this volume ?", False): + exit(0) + else: + logger.critical(f"Volume '{volume_string}' cannot be parsed. Exiting.") - def addPort(self, - port_host: int, - port_container: Union[int, str], - protocol: str = 'tcp', - host_ip: str = '0.0.0.0'): - """Add port NAT config, only applicable on bridge network mode.""" - if self.__network_host: - logger.warning("Port sharing is configured, disabling the host network mode.") - self.setNetworkMode(False) - if protocol.lower() not in ['tcp', 'udp', 'sctp']: - raise ProtocolNotSupported(f"Unknown protocol '{protocol}'") - logger.debug(f"Adding port {host_ip}:{port_host} -> {port_container}/{protocol}") - self.__ports[f"{port_container}/{protocol}"] = (host_ip, port_host) + def addUserDevice(self, user_device_config: str): + """Add a device from a user parameters""" + if EnvInfo.isDockerDesktop(): + logger.warning("Docker desktop (Windows & macOS) does not support USB device passthrough.") + logger.verbose("Official doc: https://docs.docker.com/desktop/faqs/#can-i-pass-through-a-usb-device-to-a-container") + logger.critical("Device configuration cannot be applied, aborting operation.") + self.__addDevice(user_device_config) def addRawPort(self, user_test_port: str): """Add port config from user input. @@ -1084,9 +1066,26 @@ def addRawPort(self, user_test_port: str): return self.addPort(host_port, container_port, protocol=protocol, host_ip=host_ip) - def getPorts(self) -> Dict[str, Optional[Union[int, Tuple[str, int], List[int], List[Dict[str, Union[int, str]]]]]]: - """Ports config getter""" - return self.__ports + def addRawEnv(self, env: str): + """Parse and add an environment variable from raw user input""" + key, value = self.__parseUserEnv(env) + self.addEnv(key, value) + + @classmethod + def __parseUserEnv(cls, env: str) -> Tuple[str, str]: + env_args = env.split('=') + key = env_args[0] + if len(env_args) < 2: + value = os.getenv(env, '') + if not value: + logger.critical(f"Incorrect env syntax ({env}). Please use this format: KEY=value") + else: + logger.success(f"Using system value for env {env}.") + else: + value = '='.join(env_args[1:]) + return key, value + + # ===== Display / text formatting section ===== def getTextFeatures(self, verbose: bool = False) -> str: """Text formatter for features configurations (Privileged, GUI, Network, Timezone, Shares) @@ -1113,6 +1112,19 @@ def getTextFeatures(self, verbose: bool = False) -> str: return "[i][bright_black]Default configuration[/bright_black][/i]" return result + def getVpnName(self): + """Get VPN Config name""" + if self.__vpn_path is None: + return "[bright_black]N/A[/bright_black] " + return f"[deep_sky_blue3]{self.__vpn_path.name}[/deep_sky_blue3]" + + def getTextNetworkMode(self) -> str: + """Network mode, text getter""" + network_mode = "host" if self.__network_host else "bridge" + if self.__vpn_path: + network_mode += " with VPN" + return network_mode + def getTextCreationDate(self) -> str: """Get the container creation date. If the creation date has not been supplied on the container, return empty string.""" diff --git a/exegol/utils/ContainerLogStream.py b/exegol/utils/ContainerLogStream.py index 1f1b4f29..b862887a 100644 --- a/exegol/utils/ContainerLogStream.py +++ b/exegol/utils/ContainerLogStream.py @@ -44,6 +44,7 @@ def __next__(self): if self.__data_stream is None: # The 'follow' mode cannot be used because there is no timeout mechanism and will stuck the process forever self.__data_stream = self.__container.logs(stream=True, follow=False, since=self.__since_date, until=self.__until_date) + assert self.__data_stream is not None # Parsed the data stream to extract characters and merge them into a line. for streamed_char in self.__data_stream: # When detecting an end of line, the buffer is returned as a single line. diff --git a/exegol/utils/entrypoint/EntrypointUtils.py b/exegol/utils/entrypoint/EntrypointUtils.py index c3b29241..9098f2b8 100644 --- a/exegol/utils/entrypoint/EntrypointUtils.py +++ b/exegol/utils/entrypoint/EntrypointUtils.py @@ -1,7 +1,7 @@ import io import tarfile -from exegol import ConstantConfig +from exegol.config.ConstantConfig import ConstantConfig from exegol.utils.ExeLog import logger From c479182dea22e0092057270969f8053dc4988ec0 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sun, 6 Aug 2023 18:34:49 +0200 Subject: [PATCH 009/109] Add Exegol Desktop feature --- exegol/config/UserConfig.py | 26 +++ exegol/console/TUI.py | 5 +- exegol/console/cli/ExegolCompleter.py | 10 ++ .../console/cli/actions/GenericParameters.py | 42 +++-- exegol/manager/ExegolManager.py | 10 +- exegol/model/ContainerConfig.py | 164 +++++++++++++++--- exegol/model/ExegolContainer.py | 5 +- exegol/utils/entrypoint/entrypoint.sh | 46 ++++- 8 files changed, 255 insertions(+), 53 deletions(-) diff --git a/exegol/config/UserConfig.py b/exegol/config/UserConfig.py index 8fe503c4..f0c8a3ca 100644 --- a/exegol/config/UserConfig.py +++ b/exegol/config/UserConfig.py @@ -13,6 +13,7 @@ class UserConfig(DataFileUtils, metaclass=MetaSingleton): # Static choices start_shell_options = {'zsh', 'bash', 'tmux'} shell_logging_method_options = {'script', 'asciinema'} + desktop_available_proto = {'http', 'vnc'} def __init__(self): # Defaults User config @@ -25,11 +26,15 @@ def __init__(self): self.default_start_shell: str = "zsh" self.shell_logging_method: str = "asciinema" self.shell_logging_compress: bool = True + self.desktop_default_enable: bool = False + self.desktop_default_localhost: bool = True + self.desktop_default_proto: str = "http" super().__init__("config.yml", "yml") def _build_file_content(self): config = f"""# Exegol configuration +# Full documentation: https://exegol.readthedocs.io/en/latest/exegol-wrapper/advanced-uses.html#id1 # Volume path can be changed at any time but existing containers will not be affected by the update volumes: @@ -63,6 +68,18 @@ def _build_file_content(self): # Enable automatic compression of log files (with gzip) enable_log_compression: {self.shell_logging_compress} + + # Configure your Exegol Desktop + desktop: + # Enables the desktop mode all the time + # If this attribute is set to True, then using the CLI --desktop option will be inverted and will DISABLE the desktop + enabled_by_default: {self.desktop_default_enable} + + # Default desktop protocol,can be "http", or "vnc" (additional protocols to come in the future, check online documentation for updates). + default_protocol: {self.desktop_default_proto} + + # Desktop service is exposed on localhost by default. If set to true, services will be exposed on localhost (127.0.0.1) other it will be exposed on 0.0.0.0. This setting can be overwritten with --desktop-config + localhost_by_default: {self.desktop_default_localhost} """ # TODO handle default image selection @@ -105,6 +122,12 @@ def _process_data(self): self.shell_logging_method = self._load_config_str(shell_logging_data, 'logging_method', self.shell_logging_method, choices=self.shell_logging_method_options) self.shell_logging_compress = self._load_config_bool(shell_logging_data, 'enable_log_compression', self.shell_logging_compress) + # Desktop section + desktop_data = config_data.get("desktop", {}) + self.desktop_default_enable = self._load_config_bool(desktop_data, 'enabled_by_default', self.desktop_default_enable) + self.desktop_default_proto = self._load_config_str(desktop_data, 'default_proto', self.desktop_default_proto, choices=self.desktop_available_proto) + self.desktop_default_localhost = self._load_config_bool(desktop_data, 'localhost_by_default', self.desktop_default_localhost) + def get_configs(self) -> List[str]: """User configs getter each options""" configs = [ @@ -118,6 +141,9 @@ def get_configs(self) -> List[str]: f"Default start shell: [blue]{self.default_start_shell}[/blue]", f"Shell logging method: [blue]{self.shell_logging_method}[/blue]", f"Shell logging compression: {boolFormatter(self.shell_logging_compress)}", + f"Desktop enabled by default: {boolFormatter(self.desktop_default_enable)}", + f"Desktop default protocol: [blue]{self.desktop_default_proto}[/blue]", + f"Desktop default host: [blue]{'localhost' if self.desktop_default_localhost else '0.0.0.0'}[/blue]", ] # TUI can't be called from here to avoid circular importation return configs diff --git a/exegol/console/TUI.py b/exegol/console/TUI.py index 1544ca1e..9bbb85b9 100644 --- a/exegol/console/TUI.py +++ b/exegol/console/TUI.py @@ -447,10 +447,11 @@ def __buildContainerRecapTable(container: ExegolContainerTemplate): container_info_header += f" [{color}]({container.image.getArch()})[/{color}]" recap.add_column(container_info_header) # Main features - if passwd: - recap.add_row(f"[bold blue]Credentials[/bold blue]", f"[deep_sky_blue3]{container.config.getUsername()}[/deep_sky_blue3] : [deep_sky_blue3]{passwd}[/deep_sky_blue3]") if comment: recap.add_row("[bold blue]Comment[/bold blue]", comment) + if passwd: + recap.add_row(f"[bold blue]Credentials[/bold blue]", f"[deep_sky_blue3]{container.config.getUsername()}[/deep_sky_blue3] : [deep_sky_blue3]{passwd}[/deep_sky_blue3]") + recap.add_row("[bold blue]Desktop[/bold blue]", container.config.getDesktopConfig()) if creation_date: recap.add_row("[bold blue]Creation date[/bold blue]", creation_date) recap.add_row("[bold blue]GUI[/bold blue]", boolFormatter(container.config.isGUIEnable())) diff --git a/exegol/console/cli/ExegolCompleter.py b/exegol/console/cli/ExegolCompleter.py index bd3c62bc..a5e71e3e 100644 --- a/exegol/console/cli/ExegolCompleter.py +++ b/exegol/console/cli/ExegolCompleter.py @@ -2,6 +2,7 @@ from typing import Tuple from exegol.config.DataCache import DataCache +from exegol.config.UserConfig import UserConfig from exegol.manager.UpdateManager import UpdateManager from exegol.utils.DockerUtils import DockerUtils @@ -64,6 +65,15 @@ def BuildProfileCompleter(prefix: str, parsed_args: Namespace, **kwargs) -> Tupl return tuple(data) +def DesktopConfigCompleter(prefix: str, **kwargs) -> Tuple[str, ...]: + options = list(UserConfig.desktop_available_proto) + for obj in options: + if prefix and not obj.lower().startswith(prefix.lower()): + options.remove(obj) + # TODO add interface enum + return tuple(options) + + def VoidCompleter(**kwargs) -> Tuple: """No option to auto-complet""" return () diff --git a/exegol/console/cli/actions/GenericParameters.py b/exegol/console/cli/actions/GenericParameters.py index d824348c..2d2769e4 100644 --- a/exegol/console/cli/actions/GenericParameters.py +++ b/exegol/console/cli/actions/GenericParameters.py @@ -3,7 +3,7 @@ from argcomplete.completers import EnvironCompleter, DirectoriesCompleter, FilesCompleter from exegol.config.UserConfig import UserConfig -from exegol.console.cli.ExegolCompleter import ContainerCompleter, ImageCompleter, VoidCompleter +from exegol.console.cli.ExegolCompleter import ContainerCompleter, ImageCompleter, VoidCompleter, DesktopConfigCompleter from exegol.console.cli.actions.Command import Option, GroupArg @@ -225,18 +225,6 @@ def __init__(self, groupArgs: List[GroupArg]): action="append", help="Add host [default not bold]device(s)[/default not bold] at the container creation (example: -d /dev/ttyACM0 -d /dev/bus/usb/)") - self.vpn = Option("--vpn", - dest="vpn", - default=None, - action="store", - help="Setup an OpenVPN connection at the container creation (example: --vpn /home/user/vpn/conf.ovpn)", - completer=FilesCompleter(["ovpn"], directories=True)) - self.vpn_auth = Option("--vpn-auth", - dest="vpn_auth", - default=None, - action="store", - help="Enter the credentials with a file (first line: username, second line: password) to establish the VPN connection automatically (example: --vpn-auth /home/user/vpn/auth.txt)") - self.comment = Option("--comment", dest="comment", action="store", @@ -260,6 +248,34 @@ def __init__(self, groupArgs: List[GroupArg]): {"arg": self.comment, "required": False}, title="[blue]Container creation options[/blue]")) + self.vpn = Option("--vpn", + dest="vpn", + default=None, + action="store", + help="Setup an OpenVPN connection at the container creation (example: --vpn /home/user/vpn/conf.ovpn)", + completer=FilesCompleter(["ovpn"], directories=True)) + self.vpn_auth = Option("--vpn-auth", + dest="vpn_auth", + default=None, + action="store", + help="Enter the credentials with a file (first line: username, second line: password) to establish the VPN connection automatically (example: --vpn-auth /home/user/vpn/auth.txt)") + groupArgs.append(GroupArg({"arg": self.vpn, "required": False}, {"arg": self.vpn_auth, "required": False}, title="[blue]Container creation VPN options[/blue]")) + + self.desktop = Option("--desktop", + dest="desktop", + action="store_true", + help=f"Enable or disable the Exegol desktop feature (default: {'[green]Enabled[/green]' if UserConfig().desktop_default_enable else '[red]Disabled[/red]'})") + self.desktop_config = Option("--desktop-config", + dest="desktop_config", + default="", + action="store", + help=f"Configure your exegol desktop ([blue]{', '.join(UserConfig.desktop_available_proto)}[/blue]) and its exposition " + f"(format: [blue]proto[/blue]\[:[blue]ip[/blue]\[:[blue]port[/blue]]]) " + f"(default: [blue]{UserConfig().desktop_default_proto}[/blue]:[blue]{'127.0.0.1' if UserConfig().desktop_default_localhost else '0.0.0.0'}[/blue]:[blue][/blue])", + completer=DesktopConfigCompleter) + groupArgs.append(GroupArg({"arg": self.desktop, "required": False}, + {"arg": self.desktop_config, "required": False}, + title="[blue]Container creation Desktop options[/blue]")) diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index 6c45f907..0d6d5106 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -21,7 +21,7 @@ from exegol.model.ExegolModules import ExegolModules from exegol.model.SelectableInterface import SelectableInterface from exegol.utils.DockerUtils import DockerUtils -from exegol.utils.ExeLog import logger, ExeLog, console +from exegol.utils.ExeLog import logger, ExeLog class ExegolManager: @@ -238,7 +238,6 @@ def print_sponsors(cls): """We thank [link=https://www.capgemini.com/fr-fr/carrieres/offres-emploi/][blue]Capgemini[/blue][/link] for supporting the project [bright_black](helping with dev)[/bright_black] :pray:""") logger.success("""We thank [link=https://www.hackthebox.com/][green]HackTheBox[/green][/link] for sponsoring the [bright_black]multi-arch[/bright_black] support :green_heart:""") - @classmethod def __loadOrInstallImage(cls, override_image: Optional[str] = None, @@ -460,11 +459,10 @@ def __prepareContainerConfig(cls): if ParametersManager().exegol_resources: config.enableExegolResources() if ParametersManager().log: - config.enableShellLogging() + config.enableShellLogging(ParametersManager().log_method) if ParametersManager().workspace_path: if ParametersManager().mount_current_dir: - logger.warning( - f'Workspace conflict detected (-cwd cannot be use with -w). Using: {ParametersManager().workspace_path}') + logger.warning(f'Workspace conflict detected (-cwd cannot be use with -w). Using: {ParametersManager().workspace_path}') config.setWorkspaceShare(ParametersManager().workspace_path) elif ParametersManager().mount_current_dir: config.enableCwdShare() @@ -484,6 +482,8 @@ def __prepareContainerConfig(cls): if ParametersManager().envs is not None: for env in ParametersManager().envs: config.addRawEnv(env) + if UserConfig().desktop_default_enable ^ ParametersManager().desktop: + config.enableDesktop(ParametersManager().desktop_config) if ParametersManager().comment: config.addComment(ParametersManager().comment) return config diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 396c6f7c..43f87f0b 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -4,6 +4,7 @@ import re import string from datetime import datetime +from enum import Enum from pathlib import Path, PurePath from typing import Optional, List, Dict, Union, Tuple, cast @@ -33,13 +34,24 @@ class ContainerConfig: # Reference static config data __static_gui_envs = {"_JAVA_AWT_WM_NONREPARENTING": "1", "QT_X11_NO_MITSHM": "1"} + __default_desktop_port = {"http": 6080, "vnc": 5900} - # Label features (wrapper method to enable the feature / label name) - __label_features = {"enableShellLogging": "org.exegol.feature.shell_logging"} + class ExegolFeatures(Enum): + shell_logging = "org.exegol.feature.shell_logging" + desktop = "org.exegol.feature.desktop" + + class ExegolMetadata(Enum): + creation_date = "org.exegol.metadata.creation_date" + comment = "org.exegol.metadata.comment" + password = "org.exegol.metadata.passwd" + + # Label features (label name / wrapper method to enable the feature) + __label_features = {ExegolFeatures.shell_logging.value: "enableShellLogging", + ExegolFeatures.desktop.value: "configureDesktop"} # Label metadata (label name / [setter method to set the value, getter method to update labels]) - __label_metadata = {"org.exegol.metadata.creation_date": ["setCreationDate", "getCreationDate"], - "org.exegol.metadata.comment": ["setComment", "getComment"], - "org.exegol.metadata.passwd": ["setPasswd", "getPasswd"]} + __label_metadata = {ExegolMetadata.creation_date.value: ["setCreationDate", "getCreationDate"], + ExegolMetadata.comment.value: ["setComment", "getComment"], + ExegolMetadata.password.value: ["setPasswd", "getPasswd"]} def __init__(self, container: Optional[Container] = None): """Container config default value""" @@ -71,6 +83,9 @@ def __init__(self, container: Optional[Container] = None): self.__vpn_parameters: Optional[str] = None self.__run_cmd: bool = False self.__endless_container: bool = True + self.__desktop_proto: Optional[str] = None + self.__desktop_host: Optional[str] = None + self.__desktop_port: Optional[int] = None # Metadata attributes self.__creation_date: Optional[str] = None self.__comment: Optional[str] = None @@ -140,18 +155,18 @@ def __parseLabels(self, labels: Dict[str, str]): logger.debug(f"Parsing label : {key}") if key.startswith("org.exegol.metadata."): # Find corresponding feature and attributes - for label, refs in self.__label_metadata.items(): # Setter - if label == key: - # reflective execution of setter method (set metadata value to the corresponding attribute) - getattr(self, refs[0])(value) - break + refs = self.__label_metadata.get(key) # Setter + if refs is not None: + # reflective execution of setter method (set metadata value to the corresponding attribute) + getattr(self, refs[0])(value) elif key.startswith("org.exegol.feature."): # Find corresponding feature and attributes - for attribute, label in self.__label_features.items(): - if label == key: - # reflective execution of the feature enable method (add label & set attributes) - getattr(self, attribute)() - break + enable_function = self.__label_features.get(key) + if enable_function is not None: + # reflective execution of the feature enable method (add label & set attributes) + if value == "Enabled": + value = "" + getattr(self, enable_function)(value) def __parseMounts(self, mounts: Optional[List[Dict]], name: str): """Parse Mounts object""" @@ -242,6 +257,16 @@ def interactiveConfig(self, container_name: str) -> List[str]: if not self.__enable_gui: command_options.append("--disable-X11") + # Desktop Config + if self.isDesktopEnabled(): + if Confirm("Do you want to [orange3]disable[/orange3] [blue]Desktop[/blue]?", False): + self.__disableDesktop() + elif Confirm("Do you want to [green]enable[/green] [blue]Desktop[/blue]?", False): + self.enableDesktop() + # Command builder info + if self.isDesktopEnabled(): + command_options.append("--desktop") + # Timezone config if self.__share_timezone: if Confirm("Do you want to [orange3]remove[/orange3] your [blue]shared timezone[/blue] config?", False): @@ -287,7 +312,7 @@ def interactiveConfig(self, container_name: str) -> List[str]: if Confirm("Do you want to [orange3]disable[/orange3] automatic [blue]shell logging[/blue]?", False): self.__disableShellLogging() elif Confirm("Do you want to [green]enable[/green] automatic [blue]shell logging[/blue]?", False): - self.enableShellLogging() + self.enableShellLogging(UserConfig().shell_logging_method) # Command builder info if self.__shell_logging: command_options.append("--log") @@ -417,19 +442,88 @@ def disableExegolResources(self): self.__exegol_resources = False self.removeVolume(container_path='/opt/resources') - def enableShellLogging(self): + def enableShellLogging(self, log_method: str): """Procedure to enable exegol shell logging feature""" if not self.__shell_logging: logger.verbose("Config: Enabling shell logging") self.__shell_logging = True - self.addLabel(self.__label_features.get('enableShellLogging', 'org.exegol.error'), "Enabled") + self.addLabel(self.ExegolFeatures.shell_logging.value, log_method) def __disableShellLogging(self): """Procedure to disable exegol shell logging feature""" if self.__shell_logging: logger.verbose("Config: Disabling shell logging") self.__shell_logging = False - self.removeLabel(self.__label_features.get('enableShellLogging', 'org.exegol.error')) + self.removeLabel(self.ExegolFeatures.shell_logging.value) + + def isDesktopEnabled(self): + return self.__desktop_proto is not None + + def enableDesktop(self, desktop_config: str = ""): + """Procedure to enable exegol desktop feature""" + if not self.isDesktopEnabled(): + logger.verbose("Config: Enabling exegol desktop") + self.configureDesktop(desktop_config) + assert self.__desktop_proto is not None + assert self.__desktop_host is not None + assert self.__desktop_port is not None + self.addLabel(self.ExegolFeatures.desktop.value, f"{self.__desktop_proto}:{self.__desktop_host}:{self.__desktop_port}") + # Env var are used to send these parameter to the desktop-start script + self.addEnv("DESKTOP_PROTO", self.__desktop_proto) + + if self.__network_host: + self.addEnv("DESKTOP_HOST", self.__desktop_host) + self.addEnv("DESKTOP_PORT", str(self.__desktop_port)) + else: + self.addEnv("DESKTOP_HOST", "localhost") + self.addEnv("DESKTOP_PORT", str(self.__default_desktop_port.get(self.__desktop_proto))) + # Exposing desktop service + self.addPort(port_host=self.__desktop_port, port_container=self.__default_desktop_port[self.__desktop_proto], host_ip=self.__desktop_host) + + def configureDesktop(self, desktop_config: str): + """Configure the exegol desktop feature from user parameters. + Accepted format: 'mode:host:port' + """ + self.__desktop_proto = UserConfig().desktop_default_proto + self.__desktop_host = "localhost" if UserConfig().desktop_default_localhost else "0.0.0.0" + + for i, data in enumerate(desktop_config.split(":")): + if not data: + continue + if i == 0: + data = data.lower() + if data in UserConfig.desktop_available_proto: + self.__desktop_proto = data + else: + logger.critical(f"The desktop mode '{data}' is not supported. Please choose a supported mode: [green]{', '.join(UserConfig.desktop_available_proto)}[/green].") + elif i == 1 and data: + self.__desktop_host = data + self.__desktop_port = self.__findAvailableRandomPort(self.__desktop_host) + elif i == 2: + try: + self.__desktop_port = int(data) + except ValueError: + logger.critical(f"Invalid desktop port: '{data}' is not a valid port.") + else: + logger.critical(f"Your configuration is invalid, please use the following format:[green]mode:host:port[/green]") + + if self.__desktop_port is None: + self.__desktop_port = self.__findAvailableRandomPort() + + def __disableDesktop(self): + """Procedure to disable exegol desktop feature""" + if self.isDesktopEnabled(): + logger.verbose("Config: Disabling shell logging") + assert self.__desktop_proto is not None + if not self.__network_host: + self.__removePort(self.__default_desktop_port[self.__desktop_proto]) + self.__desktop_proto = None + self.__desktop_host = None + self.__desktop_port = None + self.removeLabel(self.ExegolFeatures.desktop.value) + self.removeEnv("DESKTOP_PROTO") + self.removeEnv("DESKTOP_HOST") + self.removeEnv("DESKTOP_PORT") def enableCwdShare(self): """Procedure to share Current Working Directory with the /workspace of the container""" @@ -489,7 +583,7 @@ def addComment(self, comment): if not self.__comment: logger.verbose("Config: Adding comment to container info") self.__comment = comment - self.addLabel("org.exegol.metadata.comment", comment) + self.addLabel(self.ExegolMetadata.comment.value, comment) # ===== Functional / technical methods section ===== @@ -604,6 +698,8 @@ def getEntrypointCommand(self) -> Tuple[Optional[List[str]], Union[List[str], st entrypoint_actions = [] if self.__my_resources: entrypoint_actions.append("load_setups") + if self.isDesktopEnabled(): + entrypoint_actions.append("desktop") if self.__vpn_path is not None: entrypoint_actions.append(f"ovpn {self.__vpn_parameters}") if self.__run_cmd: @@ -611,7 +707,7 @@ def getEntrypointCommand(self) -> Tuple[Optional[List[str]], Union[List[str], st if self.__endless_container: entrypoint_actions.append("endless") else: - entrypoint_actions.append("end") + entrypoint_actions.append("finish") return self.__container_entrypoint, entrypoint_actions def getShellCommand(self) -> str: @@ -641,6 +737,16 @@ def generateRandomPassword(length: int = 30) -> str: charset = string.ascii_letters + string.digits + string.punctuation.replace("'", "") return ''.join(random.choice(charset) for i in range(length)) + @staticmethod + def __findAvailableRandomPort(interface: str = 'localhost') -> int: + """Find an available random port. Using the socket system to """ + import socket + sock = socket.socket() + sock.bind((interface, 0)) # Using port 0 let the system decide for a random port + random_port = sock.getsockname()[1] + sock.close() + return random_port + # ===== Apply config section ===== def setWorkspaceShare(self, host_directory): @@ -942,6 +1048,9 @@ def getPorts(self) -> Dict[str, Optional[Union[int, Tuple[str, int], List[int], """Ports config getter""" return self.__ports + def __removePort(self, container_port: Union[int, str], protocol: str = 'tcp'): + self.__ports.pop(f"{container_port}/{protocol}", None) + def addLabel(self, key: str, value: str): """Add a custom label to the container configuration""" self.__labels[key] = value @@ -1093,6 +1202,8 @@ def getTextFeatures(self, verbose: bool = False) -> str: result = "" if verbose or self.__privileged: result += f"{getColor(not self.__privileged)[0]}Privileged: {'On :fire:' if self.__privileged else '[green]Off :heavy_check_mark:[/green]'}{getColor(not self.__privileged)[1]}{os.linesep}" + if verbose or self.isDesktopEnabled(): + result += f"{getColor(self.isDesktopEnabled())[0]}Desktop: {self.getDesktopConfig()}{getColor(self.isDesktopEnabled())[1]}{os.linesep}" if verbose or not self.__enable_gui: result += f"{getColor(self.__enable_gui)[0]}GUI: {boolFormatter(self.__enable_gui)}{getColor(self.__enable_gui)[1]}{os.linesep}" if verbose or not self.__network_host: @@ -1112,12 +1223,19 @@ def getTextFeatures(self, verbose: bool = False) -> str: return "[i][bright_black]Default configuration[/bright_black][/i]" return result - def getVpnName(self): + def getVpnName(self) -> str: """Get VPN Config name""" if self.__vpn_path is None: return "[bright_black]N/A[/bright_black] " return f"[deep_sky_blue3]{self.__vpn_path.name}[/deep_sky_blue3]" + def getDesktopConfig(self) -> str: + """Get Desktop feature status / config""" + if not self.isDesktopEnabled(): + return boolFormatter(False) + config = f"{self.__desktop_proto}://{self.__desktop_host}:{self.__desktop_port}" + return f"[link={config}][deep_sky_blue3]{config}[/deep_sky_blue3][/link]" + def getTextNetworkMode(self) -> str: """Network mode, text getter""" network_mode = "host" if self.__network_host else "bridge" @@ -1162,7 +1280,7 @@ def getTextEnvs(self, verbose: bool = False) -> str: result = '' for k, v in self.__envs.items(): # Blacklist technical variables, only shown in verbose - if not verbose and k in list(self.__static_gui_envs.keys()) + ["DISPLAY", "PATH"]: + if not verbose and k in list(self.__static_gui_envs.keys()) + ["DISPLAY", "PATH", "DESKTOP_PROTO", "DESKTOP_HOST", "DESKTOP_PORT"]: continue result += f"{k}={v}{os.linesep}" return result diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 302b8751..8a1a47cb 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -315,4 +315,7 @@ def __updatePasswd(self): """ if self.config.getPasswd() is not None: logger.debug(f"Updating the {self.config.getUsername()} password inside the container") - self.exec(["echo", f"'{self.config.getUsername()}:{self.config.getPasswd()}'", "|", "chpasswd"], quiet=True) + self.exec(f"echo '{self.config.getUsername()}:{self.config.getPasswd()}' | chpasswd", quiet=True) + if self.config.isDesktopEnabled(): + # TODO fix passwd update + self.exec(f"echo '{self.config.getPasswd()}' | vncpasswd -f > ~/.vnc/passwd", quiet=True) diff --git a/exegol/utils/entrypoint/entrypoint.sh b/exegol/utils/entrypoint/entrypoint.sh index 7946292a..00c17f3c 100644 --- a/exegol/utils/entrypoint/entrypoint.sh +++ b/exegol/utils/entrypoint/entrypoint.sh @@ -10,7 +10,7 @@ function load_setups() { # Execute initial setup if lock file doesn't exist echo >/.exegol/.setup.lock # Run my-resources script. Logs starting with '[exegol]' will be print to the console and report back to the user through the wrapper. - if [ -f /.exegol/load_supported_setupsd.sh ]; then + if [ -f /.exegol/load_supported_setups.sh ]; then echo "Installing [green]my-resources[/green] custom setup ..." /.exegol/load_supported_setups.sh |& tee /var/log/exegol/load_setups.log | grep -i '^\[exegol]' | sed "s/^\[exegol\]\s*//gi" [ -f /var/log/exegol/load_setups.log ] && echo "Compressing [green]my-resources[/green] logs" && gzip /var/log/exegol/load_setups.log && echo "My-resources loaded" @@ -20,13 +20,13 @@ function load_setups() { fi } -function end() { +function finish() { echo "READY" } function endless() { # Start action / endless - end + finish # Entrypoint for the container, in order to have a process hanging, to keep the container alive # Alternative to running bash/zsh/whatever as entrypoint, which is longer to start and to stop and to very clean # shellcheck disable=SC2162 @@ -36,7 +36,7 @@ function endless() { function shutdown() { # Shutting down the container. # Sending SIGTERM to all interactive process for proper closing - pgrep guacd && /opt/tools/bin/desktop-stop # Stop webui desktop if started + pgrep vnc && /opt/tools/bin/desktop-stop # Stop webui desktop if started TODO improve desktop stop # shellcheck disable=SC2046 kill $(pgrep -f -- openvpn | grep -vE '^1$') 2>/dev/null # shellcheck disable=SC2046 @@ -48,7 +48,7 @@ function shutdown() { # shellcheck disable=SC2046 kill $(pgrep -x -f -- -bash) 2>/dev/null # Wait for every active process to exit (e.g: shell logging compression, VPN closing, WebUI) - wait_list="$(pgrep -f "(.log|start.sh|tomcat)" | grep -vE '^1$')" + wait_list="$(pgrep -f "(.log|start.sh|vnc)" | grep -vE '^1$')" for i in $wait_list; do # Waiting for: $i PID process to exit tail --pid="$i" -f /dev/null @@ -67,16 +67,44 @@ function _resolv_docker_host() { function ovpn() { [[ "$DISPLAY" == *"host.docker.internal"* ]] && _resolv_docker_host - # Starting openvpn as a job with '&' to be able to receive SIGTERM signal and close everything properly - echo "Starting [green]VPN[/green]" - openvpn --log-append /var/log/exegol/vpn.log "$@" & - sleep 2 # Waiting 2 seconds for the VPN to start before continuing + if ! command -v openvpn &> /dev/null + then + echo '[E]Your exegol image is not up-to-date! VPN feature is not supported!' + else + # Starting openvpn as a job with '&' to be able to receive SIGTERM signal and close everything properly + echo "Starting [green]VPN[/green]" + openvpn --log-append /var/log/exegol/vpn.log "$@" & + sleep 2 # Waiting 2 seconds for the VPN to start before continuing + fi + } function run_cmd() { /bin/zsh -c "autoload -Uz compinit; compinit; source ~/.zshrc; eval \"$CMD\"" } +function desktop() { + if [ -f /opt/tools/bin/desktop-start ] + then + echo "Starting Exegol [green]desktop[/green]" + /opt/tools/bin/desktop-start & + sleep 2 # Waiting 2 seconds for the Desktop to start before continuing + else + echo '[E]Your exegol image is not up-to-date! Desktop feature is not supported!' + fi + + #case "$mode" in + # vnc) + # echo "Start VNC" + # vncserver -localhost "yes" -rfbport "$port" -geometry "1920x1080" -SecurityTypes "VncAuth" -passwd "$HOME/.vnc/passwd" ":0" + # ;; + # http) + # echo "Start VNC" + # echo "Start websockify" + # ;; + #esac +} + ##### How "echo" works here with exegol ##### # # Every message printed here will be displayed to the console logs of the container From b1b91b1a57990fb7fa84dcb0a29b84ce3dff14dd Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 8 Aug 2023 09:46:54 +0200 Subject: [PATCH 010/109] Update desktop script + add nvnc redirect --- exegol/utils/entrypoint/entrypoint.sh | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/exegol/utils/entrypoint/entrypoint.sh b/exegol/utils/entrypoint/entrypoint.sh index 00c17f3c..28af0bd8 100644 --- a/exegol/utils/entrypoint/entrypoint.sh +++ b/exegol/utils/entrypoint/entrypoint.sh @@ -36,7 +36,7 @@ function endless() { function shutdown() { # Shutting down the container. # Sending SIGTERM to all interactive process for proper closing - pgrep vnc && /opt/tools/bin/desktop-stop # Stop webui desktop if started TODO improve desktop stop + pgrep vnc && desktop-stop # Stop webui desktop if started TODO improve desktop shutdown # shellcheck disable=SC2046 kill $(pgrep -f -- openvpn | grep -vE '^1$') 2>/dev/null # shellcheck disable=SC2046 @@ -84,25 +84,14 @@ function run_cmd() { } function desktop() { - if [ -f /opt/tools/bin/desktop-start ] + if [ -f /usr/sbin/desktop-start ] then echo "Starting Exegol [green]desktop[/green]" - /opt/tools/bin/desktop-start & + desktop-start &> /dev/null & # Disable logging sleep 2 # Waiting 2 seconds for the Desktop to start before continuing else echo '[E]Your exegol image is not up-to-date! Desktop feature is not supported!' fi - - #case "$mode" in - # vnc) - # echo "Start VNC" - # vncserver -localhost "yes" -rfbport "$port" -geometry "1920x1080" -SecurityTypes "VncAuth" -passwd "$HOME/.vnc/passwd" ":0" - # ;; - # http) - # echo "Start VNC" - # echo "Start websockify" - # ;; - #esac } ##### How "echo" works here with exegol ##### From e5626e6c63eb6ca5c4ac44c190000767d3dfd38e Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 8 Aug 2023 10:11:28 +0200 Subject: [PATCH 011/109] Fix password update --- exegol/model/ExegolContainer.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 8a1a47cb..c0faef11 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -1,7 +1,7 @@ import os import shutil from datetime import datetime -from typing import Optional, Dict, Sequence, Tuple +from typing import Optional, Dict, Sequence, Tuple, Union from docker.errors import NotFound, ImageNotFound from docker.models.containers import Container @@ -162,7 +162,7 @@ def spawnShell(self): # environment=self.config.getShellEnvs()) # logger.debug(result) - def exec(self, command: Sequence[str], as_daemon: bool = True, quiet: bool = False, is_tmp: bool = False): + def exec(self, command: Union[str, Sequence[str]], as_daemon: bool = True, quiet: bool = False, is_tmp: bool = False): """Execute a command / process on the docker container. Set as_daemon to not follow the command stream and detach the execution Set quiet to disable logs message @@ -191,14 +191,15 @@ def exec(self, command: Sequence[str], as_daemon: bool = True, quiet: bool = Fal logger.warning("Exiting this command does [red]NOT[/red] stop the process in the container") @staticmethod - def formatShellCommand(command: Sequence[str], quiet: bool = False, entrypoint_mode: bool = False) -> Tuple[str, str]: + def formatShellCommand(command: Union[str, Sequence[str]], quiet: bool = False, entrypoint_mode: bool = False) -> Tuple[str, str]: """Generic method to format a shell command and support zsh aliases. Set quiet to disable any logging here. Set entrypoint_mode to start the command with the entrypoint.sh config loader. - The first return argument is the payload to execute with every pre-routine for zsh. - The second return argument is the command itself in str format.""" # Using base64 to escape special characters - str_cmd = ' '.join(command).replace('"', '\\"') + str_cmd = command if type(command) is str else ' '.join(command) + str_cmd = str_cmd.replace('"', '\\"') if not quiet: logger.success(f"Command received: {str_cmd}") # ZSH pre-routine: Load zsh aliases and call eval to force aliases interpretation @@ -253,7 +254,7 @@ def __removeVolume(self): except PermissionError: logger.info(f"Deleting the workspace files from the [green]{self.name}[/green] container as root") # If the host can't remove the container's file and folders, the rm command is exec from the container itself as root - self.exec(["rm", "-rf", "/workspace"], as_daemon=False, quiet=True) + self.exec("rm -rf /workspace", as_daemon=False, quiet=True) try: shutil.rmtree(volume_path) except PermissionError: @@ -316,6 +317,4 @@ def __updatePasswd(self): if self.config.getPasswd() is not None: logger.debug(f"Updating the {self.config.getUsername()} password inside the container") self.exec(f"echo '{self.config.getUsername()}:{self.config.getPasswd()}' | chpasswd", quiet=True) - if self.config.isDesktopEnabled(): - # TODO fix passwd update - self.exec(f"echo '{self.config.getPasswd()}' | vncpasswd -f > ~/.vnc/passwd", quiet=True) + self.exec(f"echo '{self.config.getPasswd()}' | vncpasswd -f > ~/.vnc/passwd", quiet=True) From 98b5c8884fc6e5fdc870d7fc090aefd1f635be94 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 8 Aug 2023 11:01:15 +0200 Subject: [PATCH 012/109] Fix utf-8 encoding --- exegol/utils/WebUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/utils/WebUtils.py b/exegol/utils/WebUtils.py index 7aabee7a..66254855 100644 --- a/exegol/utils/WebUtils.py +++ b/exegol/utils/WebUtils.py @@ -92,7 +92,7 @@ def getRemoteVersion(cls, tag: str) -> Optional[str]: response = cls.__runRequest(url, service_name="Docker Registry", headers=manifest_headers, method="GET") version: Optional[str] = None if response is not None and response.status_code == 200: - data = json.loads(response.text) + data = json.loads(response.content.decode("utf-8")) # Parse metadata of the current image from v1 schema metadata = json.loads(data.get("history", [])[0]['v1Compatibility']) # Find version label and extract data From 09f00ee0dacbe86342032f6c6e68752d448f57e1 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 8 Aug 2023 15:13:20 +0200 Subject: [PATCH 013/109] Fix password update + update start desktop --- exegol/model/ContainerConfig.py | 1 + exegol/model/ExegolContainer.py | 2 +- exegol/utils/entrypoint/entrypoint.sh | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 43f87f0b..01d61d20 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -1121,6 +1121,7 @@ def addRawVolume(self, volume_string): """Add a volume to the container configuration from raw text input. Expected format is: /source/path:/target/mount:rw""" logger.debug(f"Parsing raw volume config: {volume_string}") + # TODO support relative path parsing = re.match(r'^((\w:)?([\\/][\w .,:\-|()&;]*)+):(([\\/][\w .,\-|()&;]*)+)(:(ro|rw))?$', volume_string) if parsing: diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index c0faef11..b4270ea1 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -199,7 +199,7 @@ def formatShellCommand(command: Union[str, Sequence[str]], quiet: bool = False, - The second return argument is the command itself in str format.""" # Using base64 to escape special characters str_cmd = command if type(command) is str else ' '.join(command) - str_cmd = str_cmd.replace('"', '\\"') + #str_cmd = str_cmd.replace('"', '\\"') # This fix shoudn' be necessary plus it can alter data like passwd if not quiet: logger.success(f"Command received: {str_cmd}") # ZSH pre-routine: Load zsh aliases and call eval to force aliases interpretation diff --git a/exegol/utils/entrypoint/entrypoint.sh b/exegol/utils/entrypoint/entrypoint.sh index 28af0bd8..d269bf51 100644 --- a/exegol/utils/entrypoint/entrypoint.sh +++ b/exegol/utils/entrypoint/entrypoint.sh @@ -84,10 +84,10 @@ function run_cmd() { } function desktop() { - if [ -f /usr/sbin/desktop-start ] + if command -v desktop-start &> /dev/null then - echo "Starting Exegol [green]desktop[/green]" - desktop-start &> /dev/null & # Disable logging + echo "Starting Exegol [green]desktop[/green] with [blue]${DESKTOP_PROTO}[/blue]" + desktop-start &>> ~/.vnc/startup.log # Disable logging sleep 2 # Waiting 2 seconds for the Desktop to start before continuing else echo '[E]Your exegol image is not up-to-date! Desktop feature is not supported!' From 6ad691e693db5e2b20680ee39224804805a20b1d Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 9 Aug 2023 11:42:00 -0700 Subject: [PATCH 014/109] Exegol update change branch with -v only --- exegol/manager/UpdateManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/manager/UpdateManager.py b/exegol/manager/UpdateManager.py index 1a9c4c60..aa6df1f2 100644 --- a/exegol/manager/UpdateManager.py +++ b/exegol/manager/UpdateManager.py @@ -143,7 +143,7 @@ def __updateGit(gitUtils: GitUtils) -> bool: if current_branch is None: logger.warning("HEAD is detached. Please checkout to an existing branch.") current_branch = "unknown" - if logger.isEnabledFor(ExeLog.VERBOSE) or current_branch not in ["master", "main"]: + if logger.isEnabledFor(ExeLog.VERBOSE): available_branches = gitUtils.listBranch() # Ask to checkout only if there is more than one branch available if len(available_branches) > 1: From fbbede98214121330529936341959a76f70f636e Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 9 Aug 2023 11:46:50 -0700 Subject: [PATCH 015/109] Refacto ImageSync + add start.sh --- exegol/config/ConstantConfig.py | 7 ++- exegol/model/ExegolContainer.py | 4 +- exegol/utils/entrypoint/EntrypointUtils.py | 30 ------------- exegol/utils/imgsync/ImageScriptSync.py | 43 +++++++++++++++++++ .../utils/{entrypoint => imgsync}/__init__.py | 0 .../{entrypoint => imgsync}/entrypoint.sh | 0 exegol/utils/imgsync/start.sh | 43 +++++++++++++++++++ setup.py | 5 ++- 8 files changed, 96 insertions(+), 36 deletions(-) delete mode 100644 exegol/utils/entrypoint/EntrypointUtils.py create mode 100644 exegol/utils/imgsync/ImageScriptSync.py rename exegol/utils/{entrypoint => imgsync}/__init__.py (100%) rename exegol/utils/{entrypoint => imgsync}/entrypoint.sh (100%) create mode 100644 exegol/utils/imgsync/start.sh diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index 43e3087a..a50511f6 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -15,8 +15,10 @@ class ConstantConfig: # Path of the Dockerfile build_context_path_obj: Path build_context_path: str - # Path of the Entrypoint + # Path of the Entrypoint.sh entrypoint_context_path_obj: Path + # Path of the Start.sh + start_context_path_obj: Path # Exegol config directory exegol_config_path: Path = Path().home() / ".exegol" # Docker Desktop for mac config file @@ -63,4 +65,5 @@ def findResourceContextPath(cls, resource_folder: str, source_path: str) -> Path ConstantConfig.build_context_path_obj = ConstantConfig.findResourceContextPath("exegol-docker-build", "exegol-docker-build") ConstantConfig.build_context_path = str(ConstantConfig.build_context_path_obj) -ConstantConfig.entrypoint_context_path_obj = ConstantConfig.findResourceContextPath("exegol-entrypoint", "exegol/utils/entrypoint/entrypoint.sh") +ConstantConfig.entrypoint_context_path_obj = ConstantConfig.findResourceContextPath("exegol-imgsync", "exegol/utils/imgsync/entrypoint.sh") +ConstantConfig.start_context_path_obj = ConstantConfig.findResourceContextPath("exegol-imgsync", "exegol/utils/imgsync/start.sh") diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index b4270ea1..7c9e7a05 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -15,7 +15,7 @@ from exegol.model.SelectableInterface import SelectableInterface from exegol.utils.ContainerLogStream import ContainerLogStream from exegol.utils.ExeLog import logger, console -from exegol.utils.entrypoint.EntrypointUtils import getEntrypointTarData +from exegol.utils.imgsync.ImageScriptSync import getImageSyncTarData class ExegolContainer(ExegolContainerTemplate, SelectableInterface): @@ -281,7 +281,7 @@ def postCreateSetup(self, is_temporary: bool = False): # if not a temporary container, apply custom config if not is_temporary: # Update entrypoint script in the container - self.__container.put_archive("/", getEntrypointTarData()) + self.__container.put_archive("/", getImageSyncTarData()) if self.__container.status.lower() == "created": self.__start_container() self.__updatePasswd() diff --git a/exegol/utils/entrypoint/EntrypointUtils.py b/exegol/utils/entrypoint/EntrypointUtils.py deleted file mode 100644 index 9098f2b8..00000000 --- a/exegol/utils/entrypoint/EntrypointUtils.py +++ /dev/null @@ -1,30 +0,0 @@ -import io -import tarfile - -from exegol.config.ConstantConfig import ConstantConfig -from exegol.utils.ExeLog import logger - - -def getEntrypointTarData(): - """The purpose of this class is to generate and overwrite the entrypoint script of exegol containers - to integrate the latest features, whatever the version of the image.""" - - # Load entrypoint data - script_path = ConstantConfig.entrypoint_context_path_obj - logger.debug(f"Entrypoint path: {str(script_path)}") - if not script_path.is_file(): - logger.error("Unable to find the entrypoint script! Your Exegol installation is probably broken...") - return None - with open(script_path, 'rb') as f: - raw = f.read() - data = io.BytesIO(initial_bytes=raw) - - # Create tar file - stream = io.BytesIO() - with tarfile.open(fileobj=stream, mode='w|') as entry_tar: - # Import file to tar object - info = tarfile.TarInfo(name="/.exegol/entrypoint.sh") - info.size = len(raw) - info.mode = 0o500 - entry_tar.addfile(info, fileobj=data) - return stream.getvalue() diff --git a/exegol/utils/imgsync/ImageScriptSync.py b/exegol/utils/imgsync/ImageScriptSync.py new file mode 100644 index 00000000..bb603ebd --- /dev/null +++ b/exegol/utils/imgsync/ImageScriptSync.py @@ -0,0 +1,43 @@ +import io +import tarfile + +from exegol.config.ConstantConfig import ConstantConfig +from exegol.utils.ExeLog import logger + + +def getImageSyncTarData(): + """The purpose of this class is to generate and overwrite scripts like the entrypoint or start.sh inside exegol containers + to integrate the latest features, whatever the version of the image.""" + + entrypoint_script_path = ConstantConfig.entrypoint_context_path_obj + logger.debug(f"Entrypoint script path: {str(entrypoint_script_path)}") + start_script_path = ConstantConfig.start_context_path_obj + logger.debug(f"Start script path: {str(start_script_path)}") + if not entrypoint_script_path.is_file() or not start_script_path.is_file(): + logger.error("Unable to find the entrypoint or start script! Your Exegol installation is probably broken...") + return None + # Create tar file + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode='w|') as entry_tar: + # Load entrypoint data + with open(entrypoint_script_path, 'rb') as f: + raw = f.read() + data = io.BytesIO(initial_bytes=raw) + + # Import file to tar object + info = tarfile.TarInfo(name="/.exegol/entrypoint.sh") + info.size = len(raw) + info.mode = 0o500 + entry_tar.addfile(info, fileobj=data) + + # Load start data + with open(start_script_path, 'rb') as f: + raw = f.read() + data = io.BytesIO(initial_bytes=raw) + + # Import file to tar object + info = tarfile.TarInfo(name="/.exegol/start.sh") + info.size = len(raw) + info.mode = 0o500 + entry_tar.addfile(info, fileobj=data) + return stream.getvalue() diff --git a/exegol/utils/entrypoint/__init__.py b/exegol/utils/imgsync/__init__.py similarity index 100% rename from exegol/utils/entrypoint/__init__.py rename to exegol/utils/imgsync/__init__.py diff --git a/exegol/utils/entrypoint/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh similarity index 100% rename from exegol/utils/entrypoint/entrypoint.sh rename to exegol/utils/imgsync/entrypoint.sh diff --git a/exegol/utils/imgsync/start.sh b/exegol/utils/imgsync/start.sh new file mode 100644 index 00000000..70700774 --- /dev/null +++ b/exegol/utils/imgsync/start.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +function shell_logging() { + # First parameter is the method to use for shell logging (default to script) + method=$1 + # The second parameter is the shell command to use for the user + user_shell=$2 + # The third enable compression at the end of the session + compress=$3 + + # Logging shell using $method and spawn a $user_shell shell + + umask 007 + mkdir -p /workspace/logs/ + filelog="/workspace/logs/$(date +%d-%m-%Y_%H-%M-%S)_shell.${method}" + + case $method in + "asciinema") + # echo "Run using asciinema" + asciinema rec -i 2 --stdin --quiet --command "$user_shell" --title "$(hostname | sed 's/^exegol-/\[EXEGOL\] /') $(date '+%d/%m/%Y %H:%M:%S')" "$filelog" + ;; + + "script") + # echo "Run using script" + script -qefac "$user_shell" "$filelog" + ;; + + *) + echo "Unknown '$method' shell logging method, using 'script' as default shell logging method." + script -qefac "$user_shell" "$filelog" + ;; + esac + + if [ "$compress" = 'True' ]; then + echo 'Compressing logs, please wait...' + gzip "$filelog" + fi + exit 0 +} + +$@ || (echo -e "[!] This version of the image ($(cat /opt/.exegol_version || echo '?')) does not support the $1 feature.\n[*] Please update your image and create a new container with before using this new feature."; exit 1) + +exit 0 diff --git a/setup.py b/setup.py index e0012926..7c438d83 100644 --- a/setup.py +++ b/setup.py @@ -23,8 +23,9 @@ if data_files_dict.get(key) is None: data_files_dict[key] = [] data_files_dict[key].append(str(path)) -## exegol-entrypoint script -data_files_dict["exegol-entrypoint"] = ["exegol/utils/entrypoint/entrypoint.sh"] +## exegol scripts pushed from the wrapper +data_files_dict["exegol-imgsync"] = ["exegol/utils/imgsync/entrypoint.sh"] +data_files_dict["exegol-imgsync"] = ["exegol/utils/imgsync/start.sh"] # Dict to tuple for k, v in data_files_dict.items(): From 2e33ef9b0730cc4526935ffebefa45013eb710ab Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 9 Aug 2023 12:18:10 -0700 Subject: [PATCH 016/109] start.sh v2 with shell logging support --- exegol/model/ContainerConfig.py | 27 +++++++++++---------------- exegol/utils/imgsync/start.sh | 10 +++++++++- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 01d61d20..c3c12ac9 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -712,22 +712,8 @@ def getEntrypointCommand(self) -> Tuple[Optional[List[str]], Union[List[str], st def getShellCommand(self) -> str: """Get container command for opening a new shell""" - # If shell logging was enabled at container creation, it'll always be enabled for every shell. - # If not, it can be activated per shell basis - if self.__shell_logging or ParametersManager().log: - if self.__start_delegate_mode: - # Use a start.sh script to handle the feature with the tools and feature corresponding to the image version - # Start shell_logging feature using the user's specified method with the configured default shell w/ or w/o compression at the end - return f"/.exegol/start.sh shell_logging {ParametersManager().log_method} {ParametersManager().shell} {UserConfig().shell_logging_compress ^ ParametersManager().log_compress}" - else: - # Legacy command support - if ParametersManager().log_method != "script": - logger.warning("Your image version does not allow customization of the shell logging method. Using legacy script method.") - compression_cmd = '' - if UserConfig().shell_logging_compress ^ ParametersManager().log_compress: - compression_cmd = 'echo "Compressing logs, please wait..."; gzip $filelog; ' - return f"bash -c 'umask 007; mkdir -p /workspace/logs/; filelog=/workspace/logs/$(date +%d-%m-%Y_%H-%M-%S)_shell.log; script -qefac {ParametersManager().shell} $filelog; {compression_cmd}exit'" - return ParametersManager().shell + # Use a start.sh script to handle features with the wrapper + return "/.exegol/start.sh" @staticmethod def generateRandomPassword(length: int = 30) -> str: @@ -1012,6 +998,9 @@ def getEnvs(self) -> Dict[str, str]: def getShellEnvs(self) -> List[str]: """Overriding envs when opening a shell""" result = [] + # Select default shell to use + result.append(f"START_SHELL={ParametersManager().shell}") + # Share GUI Display config if self.__enable_gui: current_display = GuiUtils.getDisplayEnv() # If the default DISPLAY environment in the container is not the same as the DISPLAY of the user's session, @@ -1021,6 +1010,12 @@ def getShellEnvs(self) -> List[str]: # but exegol can be launched from remote access via ssh with X11 forwarding # (Be careful, an .Xauthority file may be needed). result.append(f"DISPLAY={current_display}") + # Handle shell logging + # If shell logging was enabled at container creation, it'll always be enabled for every shell. + # If not, it can be activated per shell basic + if self.__shell_logging or ParametersManager().log: + result.append(f"START_SHELL_LOGGING={ParametersManager().log_method}") + result.append(f"START_SHELL_COMPRESS={UserConfig().shell_logging_compress ^ ParametersManager().log_compress}") # Overwrite env from user parameters user_envs = ParametersManager().envs if user_envs is not None: diff --git a/exegol/utils/imgsync/start.sh b/exegol/utils/imgsync/start.sh index 70700774..f70a0b1c 100644 --- a/exegol/utils/imgsync/start.sh +++ b/exegol/utils/imgsync/start.sh @@ -38,6 +38,14 @@ function shell_logging() { exit 0 } -$@ || (echo -e "[!] This version of the image ($(cat /opt/.exegol_version || echo '?')) does not support the $1 feature.\n[*] Please update your image and create a new container with before using this new feature."; exit 1) +# Find default user shell to use from env var +user_shell=${START_SHELL:-"/bin/zsh"} + +# If shell logging is enable, the method to use is stored in env var +if [ "$START_SHELL_LOGGING" ]; then + shell_logging "$START_SHELL_LOGGING" "$user_shell" "$START_SHELL_COMPRESS" +else + $user_shell +fi exit 0 From 894c6ccac14830351247d18df79179f53b228909 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 9 Aug 2023 12:22:46 -0700 Subject: [PATCH 017/109] Patch default shell with entrypointv2 --- exegol/utils/imgsync/entrypoint.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh index d269bf51..9fb5f1bd 100644 --- a/exegol/utils/imgsync/entrypoint.sh +++ b/exegol/utils/imgsync/entrypoint.sh @@ -2,6 +2,10 @@ # SIGTERM received (the container is stopping, every process must be gracefully stopped before the timeout). trap shutdown SIGTERM +function exegol_init() { + usermod -s "/.exegol/start.sh" root # TODO review +} + # Function specific function load_setups() { # Load custom setups (supported setups, and user setup) @@ -103,6 +107,7 @@ function desktop() { # ############################################# echo "Starting exegol" +exegol_init ### Argument parsing From b4c3da1342f5051a77b15a960fd4436b32e2378c Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 9 Aug 2023 20:05:44 -0700 Subject: [PATCH 018/109] Improve start.sh shell logging --- exegol/manager/ExegolManager.py | 3 +- exegol/model/ContainerConfig.py | 44 ++++++++++++++++++------------ exegol/utils/imgsync/entrypoint.sh | 2 +- exegol/utils/imgsync/start.sh | 8 ++++++ 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index 0d6d5106..d6c2b3a7 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -459,7 +459,8 @@ def __prepareContainerConfig(cls): if ParametersManager().exegol_resources: config.enableExegolResources() if ParametersManager().log: - config.enableShellLogging(ParametersManager().log_method) + config.enableShellLogging(ParametersManager().log_method, + UserConfig().shell_logging_compress ^ ParametersManager().log_compress) if ParametersManager().workspace_path: if ParametersManager().mount_current_dir: logger.warning(f'Workspace conflict detected (-cwd cannot be use with -w). Using: {ParametersManager().workspace_path}') diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index c3c12ac9..28350b30 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -45,6 +45,14 @@ class ExegolMetadata(Enum): comment = "org.exegol.metadata.comment" password = "org.exegol.metadata.passwd" + class ExegolEnv(Enum): + user_shell = "START_SHELL" + shell_logging_method = "START_SHELL_LOGGING" + shell_logging_compress = "START_SHELL_COMPRESS" + desktop_protocol = "DESKTOP_PROTO" + desktop_host = "DESKTOP_HOST" + desktop_port = "DESKTOP_PORT" + # Label features (label name / wrapper method to enable the feature) __label_features = {ExegolFeatures.shell_logging.value: "enableShellLogging", ExegolFeatures.desktop.value: "configureDesktop"} @@ -78,7 +86,6 @@ def __init__(self, container: Optional[Container] = None): self.__container_entrypoint: List[str] = self.__default_entrypoint self.__vpn_path: Optional[Union[Path, PurePath]] = None self.__shell_logging: bool = False - self.__start_delegate_mode: bool = False # Entrypoint features self.__vpn_parameters: Optional[str] = None self.__run_cmd: bool = False @@ -107,8 +114,6 @@ def __parseContainerConfig(self, container: Container): self.__parseEnvs(container_config.get("Env", [])) self.__parseLabels(container_config.get("Labels", {})) self.interactive = container_config.get("OpenStdin", True) - # If entrypoint is set on the image, considering the presence of start.sh script for delegates features - self.__start_delegate_mode = container.attrs['Config']['Entrypoint'] is not None self.__enable_gui = False for env in self.__envs: if "DISPLAY" in env: @@ -312,7 +317,7 @@ def interactiveConfig(self, container_name: str) -> List[str]: if Confirm("Do you want to [orange3]disable[/orange3] automatic [blue]shell logging[/blue]?", False): self.__disableShellLogging() elif Confirm("Do you want to [green]enable[/green] automatic [blue]shell logging[/blue]?", False): - self.enableShellLogging(UserConfig().shell_logging_method) + self.enableShellLogging(UserConfig().shell_logging_method, UserConfig().shell_logging_compress) # Command builder info if self.__shell_logging: command_options.append("--log") @@ -442,11 +447,14 @@ def disableExegolResources(self): self.__exegol_resources = False self.removeVolume(container_path='/opt/resources') - def enableShellLogging(self, log_method: str): + def enableShellLogging(self, log_method: str, compress_mode: Optional[bool] = None): """Procedure to enable exegol shell logging feature""" if not self.__shell_logging: logger.verbose("Config: Enabling shell logging") self.__shell_logging = True + self.addEnv(self.ExegolEnv.shell_logging_method.value, log_method) + if compress_mode is not None: + self.addEnv(self.ExegolEnv.shell_logging_compress.value, str(compress_mode)) self.addLabel(self.ExegolFeatures.shell_logging.value, log_method) def __disableShellLogging(self): @@ -454,6 +462,8 @@ def __disableShellLogging(self): if self.__shell_logging: logger.verbose("Config: Disabling shell logging") self.__shell_logging = False + self.removeEnv(self.ExegolEnv.shell_logging_method.value) + self.removeEnv(self.ExegolEnv.shell_logging_compress.value) self.removeLabel(self.ExegolFeatures.shell_logging.value) def isDesktopEnabled(self): @@ -469,14 +479,14 @@ def enableDesktop(self, desktop_config: str = ""): assert self.__desktop_port is not None self.addLabel(self.ExegolFeatures.desktop.value, f"{self.__desktop_proto}:{self.__desktop_host}:{self.__desktop_port}") # Env var are used to send these parameter to the desktop-start script - self.addEnv("DESKTOP_PROTO", self.__desktop_proto) + self.addEnv(self.ExegolEnv.desktop_protocol.value, self.__desktop_proto) if self.__network_host: - self.addEnv("DESKTOP_HOST", self.__desktop_host) - self.addEnv("DESKTOP_PORT", str(self.__desktop_port)) + self.addEnv(self.ExegolEnv.desktop_host.value, self.__desktop_host) + self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__desktop_port)) else: - self.addEnv("DESKTOP_HOST", "localhost") - self.addEnv("DESKTOP_PORT", str(self.__default_desktop_port.get(self.__desktop_proto))) + self.addEnv(self.ExegolEnv.desktop_host.value, "localhost") + self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__default_desktop_port.get(self.__desktop_proto))) # Exposing desktop service self.addPort(port_host=self.__desktop_port, port_container=self.__default_desktop_port[self.__desktop_proto], host_ip=self.__desktop_host) @@ -521,9 +531,9 @@ def __disableDesktop(self): self.__desktop_host = None self.__desktop_port = None self.removeLabel(self.ExegolFeatures.desktop.value) - self.removeEnv("DESKTOP_PROTO") - self.removeEnv("DESKTOP_HOST") - self.removeEnv("DESKTOP_PORT") + self.removeEnv(self.ExegolEnv.desktop_protocol.value) + self.removeEnv(self.ExegolEnv.desktop_host.value) + self.removeEnv(self.ExegolEnv.desktop_port.value) def enableCwdShare(self): """Procedure to share Current Working Directory with the /workspace of the container""" @@ -999,7 +1009,7 @@ def getShellEnvs(self) -> List[str]: """Overriding envs when opening a shell""" result = [] # Select default shell to use - result.append(f"START_SHELL={ParametersManager().shell}") + result.append(f"{self.ExegolEnv.user_shell.value}={ParametersManager().shell}") # Share GUI Display config if self.__enable_gui: current_display = GuiUtils.getDisplayEnv() @@ -1014,8 +1024,8 @@ def getShellEnvs(self) -> List[str]: # If shell logging was enabled at container creation, it'll always be enabled for every shell. # If not, it can be activated per shell basic if self.__shell_logging or ParametersManager().log: - result.append(f"START_SHELL_LOGGING={ParametersManager().log_method}") - result.append(f"START_SHELL_COMPRESS={UserConfig().shell_logging_compress ^ ParametersManager().log_compress}") + result.append(f"{self.ExegolEnv.shell_logging_method.value}={ParametersManager().log_method}") + result.append(f"{self.ExegolEnv.shell_logging_compress.value}={UserConfig().shell_logging_compress ^ ParametersManager().log_compress}") # Overwrite env from user parameters user_envs = ParametersManager().envs if user_envs is not None: @@ -1276,7 +1286,7 @@ def getTextEnvs(self, verbose: bool = False) -> str: result = '' for k, v in self.__envs.items(): # Blacklist technical variables, only shown in verbose - if not verbose and k in list(self.__static_gui_envs.keys()) + ["DISPLAY", "PATH", "DESKTOP_PROTO", "DESKTOP_HOST", "DESKTOP_PORT"]: + if not verbose and k in list(self.__static_gui_envs.keys()) + [v.value for v in self.ExegolEnv] + ["DISPLAY", "PATH"]: continue result += f"{k}={v}{os.linesep}" return result diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh index 9fb5f1bd..413f8faf 100644 --- a/exegol/utils/imgsync/entrypoint.sh +++ b/exegol/utils/imgsync/entrypoint.sh @@ -3,7 +3,7 @@ trap shutdown SIGTERM function exegol_init() { - usermod -s "/.exegol/start.sh" root # TODO review + usermod -s "/.exegol/start.sh" root } # Function specific diff --git a/exegol/utils/imgsync/start.sh b/exegol/utils/imgsync/start.sh index f70a0b1c..f1eb0c7d 100644 --- a/exegol/utils/imgsync/start.sh +++ b/exegol/utils/imgsync/start.sh @@ -8,6 +8,14 @@ function shell_logging() { # The third enable compression at the end of the session compress=$3 + # Test if the command is supported on the current image + if ! command -v "$method" &> /dev/null + then + echo "Shell logging with $method is not supported by this image version, try with a newer one." + $user_shell + exit 0 + fi + # Logging shell using $method and spawn a $user_shell shell umask 007 From 3ba77fe30c27651c2ab15b9e43b33938a3260e2f Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 9 Aug 2023 20:15:00 -0700 Subject: [PATCH 019/109] Update stop timeout --- exegol/manager/ExegolManager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index d6c2b3a7..ec0d62d0 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -108,7 +108,7 @@ def stop(cls): container = cls.__loadOrCreateContainer(multiple=True, must_exist=True) assert container is not None and type(container) is list for c in container: - c.stop(timeout=2) + c.stop(timeout=5) @classmethod def restart(cls): From d19688d0070632abe12bf8b66830cc2c488f5cb1 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 9 Aug 2023 21:56:22 -0700 Subject: [PATCH 020/109] Bump next test version --- tests/test_exegol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_exegol.py b/tests/test_exegol.py index 1687cd35..442eb5b8 100644 --- a/tests/test_exegol.py +++ b/tests/test_exegol.py @@ -2,4 +2,4 @@ def test_version(): - assert __version__ == '4.2.5' + assert __version__ == '4.2.6' From 4380b431b934629ff90ced097b5f933df62fad0c Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 9 Aug 2023 21:58:06 -0700 Subject: [PATCH 021/109] Upgrade beta version for desktop & entrypoint version --- exegol/config/ConstantConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index f12bd2ae..edb37c03 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -5,7 +5,7 @@ class ConstantConfig: """Constant parameters information""" # Exegol Version - version: str = "4.2.5" + version: str = "4.3.0b1" # Exegol documentation link documentation: str = "https://exegol.rtfd.io/" From cb2f7432bc8198f3917c0a48bb2f9f35425a5899 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Thu, 10 Aug 2023 06:45:21 -0700 Subject: [PATCH 022/109] Changing error messages for unsupported features --- exegol/utils/imgsync/entrypoint.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh index 413f8faf..12caf2a9 100644 --- a/exegol/utils/imgsync/entrypoint.sh +++ b/exegol/utils/imgsync/entrypoint.sh @@ -73,7 +73,7 @@ function ovpn() { [[ "$DISPLAY" == *"host.docker.internal"* ]] && _resolv_docker_host if ! command -v openvpn &> /dev/null then - echo '[E]Your exegol image is not up-to-date! VPN feature is not supported!' + echo '[E]Your exegol image does not support the VPN feature' else # Starting openvpn as a job with '&' to be able to receive SIGTERM signal and close everything properly echo "Starting [green]VPN[/green]" @@ -94,7 +94,7 @@ function desktop() { desktop-start &>> ~/.vnc/startup.log # Disable logging sleep 2 # Waiting 2 seconds for the Desktop to start before continuing else - echo '[E]Your exegol image is not up-to-date! Desktop feature is not supported!' + echo '[E]Your exegol image does not support the Desktop features' fi } From afab6b776946d72025556bb8e3c1d7db9b7c384f Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 10 Aug 2023 08:48:54 -0700 Subject: [PATCH 023/109] Use IP config instead of localhost dns name --- exegol/model/ContainerConfig.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 28350b30..8292d942 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -495,7 +495,7 @@ def configureDesktop(self, desktop_config: str): Accepted format: 'mode:host:port' """ self.__desktop_proto = UserConfig().desktop_default_proto - self.__desktop_host = "localhost" if UserConfig().desktop_default_localhost else "0.0.0.0" + self.__desktop_host = "127.0.0.1" if UserConfig().desktop_default_localhost else "0.0.0.0" for i, data in enumerate(desktop_config.split(":")): if not data: @@ -1239,7 +1239,8 @@ def getDesktopConfig(self) -> str: """Get Desktop feature status / config""" if not self.isDesktopEnabled(): return boolFormatter(False) - config = f"{self.__desktop_proto}://{self.__desktop_host}:{self.__desktop_port}" + config = (f"{self.__desktop_proto}://" + f"{'localhost' if self.__desktop_host == '127.0.0.1' else self.__desktop_host}:{self.__desktop_port}") return f"[link={config}][deep_sky_blue3]{config}[/deep_sky_blue3][/link]" def getTextNetworkMode(self) -> str: From 2696e8a711dc7c83716d0320fd81d510133c537b Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Thu, 10 Aug 2023 09:11:26 -0700 Subject: [PATCH 024/109] Replacing GUI texts with X11 --- exegol/console/TUI.py | 2 +- .../console/cli/actions/GenericParameters.py | 2 +- exegol/model/ContainerConfig.py | 16 +++++----- exegol/model/ExegolContainer.py | 2 +- exegol/utils/GuiUtils.py | 32 +++++++++---------- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/exegol/console/TUI.py b/exegol/console/TUI.py index e8eae5f0..100b154c 100644 --- a/exegol/console/TUI.py +++ b/exegol/console/TUI.py @@ -454,7 +454,7 @@ def __buildContainerRecapTable(container: ExegolContainerTemplate): recap.add_row("[bold blue]Desktop[/bold blue]", container.config.getDesktopConfig()) if creation_date: recap.add_row("[bold blue]Creation date[/bold blue]", creation_date) - recap.add_row("[bold blue]GUI[/bold blue]", boolFormatter(container.config.isGUIEnable())) + recap.add_row("[bold blue]X11[/bold blue]", boolFormatter(container.config.isGUIEnable())) recap.add_row("[bold blue]Network[/bold blue]", container.config.getTextNetworkMode()) recap.add_row("[bold blue]Timezone[/bold blue]", boolFormatter(container.config.isTimezoneShared())) recap.add_row("[bold blue]Exegol resources[/bold blue]", boolFormatter(container.config.isExegolResourcesEnable()) + diff --git a/exegol/console/cli/actions/GenericParameters.py b/exegol/console/cli/actions/GenericParameters.py index 2d2769e4..cab30f6c 100644 --- a/exegol/console/cli/actions/GenericParameters.py +++ b/exegol/console/cli/actions/GenericParameters.py @@ -150,7 +150,7 @@ def __init__(self, groupArgs: List[GroupArg]): action="store_false", default=True, dest="X11", - help="Disable display sharing to run GUI-based applications (default: [green]Enabled[/green])") + help="Disable X11 sharing to run GUI-based applications (default: [green]Enabled[/green])") self.my_resources = Option("--disable-my-resources", action="store_false", default=True, diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 8292d942..fb1a3250 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -252,11 +252,11 @@ def interactiveConfig(self, container_name: str) -> List[str]: self.setWorkspaceShare(workspace_path) command_options.append(f"-w {workspace_path}") - # GUI Config + # X11 sharing (GUI) config if self.__enable_gui: - if Confirm("Do you want to [orange3]disable[/orange3] [blue]GUI[/blue]?", False): + if Confirm("Do you want to [orange3]disable[/orange3] [blue]X11[/blue] (i.e. GUI apps)?", False): self.__disableGUI() - elif Confirm("Do you want to [green]enable[/green] [blue]GUI[/blue]?", False): + elif Confirm("Do you want to [green]enable[/green] [blue]X11[/blue] (i.e. GUI apps)?", False): self.enableGUI() # Command builder info if not self.__enable_gui: @@ -343,7 +343,7 @@ def interactiveConfig(self, container_name: str) -> List[str]: def enableGUI(self): """Procedure to enable GUI feature""" if not GuiUtils.isGuiAvailable(): - logger.error("GUI feature is [red]not available[/red] on your environment. [orange3]Skipping[/orange3].") + logger.error("X11 feature (i.e. GUI apps) is [red]not available[/red] on your environment. [orange3]Skipping[/orange3].") return if not self.__enable_gui: logger.verbose("Config: Enabling display sharing") @@ -361,7 +361,7 @@ def enableGUI(self): self.__enable_gui = True def __disableGUI(self): - """Procedure to enable GUI feature (Only for interactive config)""" + """Procedure to disable X11 (GUI) feature (Only for interactive config)""" if self.__enable_gui: self.__enable_gui = False logger.verbose("Config: Disabling display sharing") @@ -1010,7 +1010,7 @@ def getShellEnvs(self) -> List[str]: result = [] # Select default shell to use result.append(f"{self.ExegolEnv.user_shell.value}={ParametersManager().shell}") - # Share GUI Display config + # Share X11 (GUI Display) config if self.__enable_gui: current_display = GuiUtils.getDisplayEnv() # If the default DISPLAY environment in the container is not the same as the DISPLAY of the user's session, @@ -1203,7 +1203,7 @@ def __parseUserEnv(cls, env: str) -> Tuple[str, str]: # ===== Display / text formatting section ===== def getTextFeatures(self, verbose: bool = False) -> str: - """Text formatter for features configurations (Privileged, GUI, Network, Timezone, Shares) + """Text formatter for features configurations (Privileged, X11, Network, Timezone, Shares) Print config only if they are different from their default config (or print everything in verbose mode)""" result = "" if verbose or self.__privileged: @@ -1211,7 +1211,7 @@ def getTextFeatures(self, verbose: bool = False) -> str: if verbose or self.isDesktopEnabled(): result += f"{getColor(self.isDesktopEnabled())[0]}Desktop: {self.getDesktopConfig()}{getColor(self.isDesktopEnabled())[1]}{os.linesep}" if verbose or not self.__enable_gui: - result += f"{getColor(self.__enable_gui)[0]}GUI: {boolFormatter(self.__enable_gui)}{getColor(self.__enable_gui)[1]}{os.linesep}" + result += f"{getColor(self.__enable_gui)[0]}X11: {boolFormatter(self.__enable_gui)}{getColor(self.__enable_gui)[1]}{os.linesep}" if verbose or not self.__network_host: result += f"[green]Network mode: [/green]{self.getTextNetworkMode()}{os.linesep}" if self.__vpn_path is not None: diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 3666d39a..5e6b8205 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -291,7 +291,7 @@ def postCreateSetup(self, is_temporary: bool = False): def __applyXhostACL(self): """ - If GUI is enabled, allow X11 access on host ACL (if not already allowed) for linux and mac. + If X11 (GUI) is enabled, allow X11 access on host ACL (if not already allowed) for linux and mac. On Windows host, WSLg X11 don't have xhost ACL. :return: """ diff --git a/exegol/utils/GuiUtils.py b/exegol/utils/GuiUtils.py index a597428e..3bf6be83 100644 --- a/exegol/utils/GuiUtils.py +++ b/exegol/utils/GuiUtils.py @@ -15,7 +15,7 @@ class GuiUtils: """This utility class allows determining if the current system supports the GUI - from the information of the system.""" + from the information of the system (through X11 sharing).""" __distro_name = "" default_x11_path = "/tmp/.X11-unix" @@ -26,7 +26,7 @@ def isGuiAvailable(cls) -> bool: Check if the host OS can support GUI application with X11 sharing :return: bool """ - # GUI was not supported on Windows before WSLg + # GUI (X11 sharing) was not supported on Windows before WSLg if EnvInfo.isWindowsHost(): return cls.__windowsGuiChecks() elif EnvInfo.isMacHost(): @@ -48,9 +48,9 @@ def getX11SocketPath(cls) -> Optional[str]: # Mount point from a WSL shell context return f"/mnt/wslg/.X11-unix" else: - # From a Windows context, a WSL distro should have been supply during GUI checks + # From a Windows context, a WSL distro should have been supply during GUI (X11 sharing) checks logger.debug(f"No WSL distro have been previously found: '{cls.__distro_name}'") - raise CancelOperation("Exegol tried to create a container with GUI support on a Windows host " + raise CancelOperation("Exegol tried to create a container with X11 sharing on a Windows host " "without having performed the availability tests before.") elif EnvInfo.isMacHost(): # Docker desktop don't support UNIX socket through volume, we are using XQuartz over the network until then @@ -68,10 +68,10 @@ def getDisplayEnv(cls) -> str: # xquartz Mac mode return "host.docker.internal:0" - # Add ENV check is case of user don't have it, which will mess up GUI if fallback does not work + # Add ENV check is case of user don't have it, which will mess up GUI (X11 sharing) if fallback does not work # @see https://github.com/ThePorgs/Exegol/issues/148 if os.getenv("DISPLAY") is None: - logger.warning("The DISPLAY environment variable is not set on your host. This can prevent GUI apps to start") + logger.warning("The DISPLAY environment variable is not set on your host. This can prevent GUI apps to start through X11 sharing") # DISPLAY var is fetch from the current user environment. If it doesn't exist, using ':0'. return os.getenv('DISPLAY', ":0") @@ -81,7 +81,7 @@ def getDisplayEnv(cls) -> str: @classmethod def __macGuiChecks(cls) -> bool: """ - Procedure to check if the Mac host supports GUI with docker through XQuartz + Procedure to check if the Mac host supports GUI (X11 sharing) with docker through XQuartz :return: bool """ if not cls.__isXQuartzInstalled(): @@ -170,27 +170,27 @@ def __startXQuartz() -> bool: @classmethod def __windowsGuiChecks(cls) -> bool: """ - Procedure to check if the Windows host supports GUI with docker through WSLg + Procedure to check if the Windows host supports GUI (X11 sharing) with docker through WSLg :return: bool """ logger.debug("Testing WSLg availability") - # WSL + WSLg must be available on the Windows host for the GUI to work + # WSL + WSLg must be available on the Windows host for the GUI to work through X11 sharing if not cls.__wsl_available(): - logger.error("WSL is [orange3]not available[/orange3] on your system. GUI is not supported.") + logger.error("WSL is [orange3]not available[/orange3] on your system. X11 sharing is not supported.") return False # Only WSL2 support WSLg if EnvInfo.getDockerEngine() != EnvInfo.DockerEngine.WLS2: - logger.error("Docker must be run with [orange3]WSL2[/orange3] engine in order to support GUI applications.") + logger.error("Docker must be run with [orange3]WSL2[/orange3] engine in order to support X11 sharing (i.e. GUI apps).") return False logger.debug("WSL is [green]available[/green] and docker is using WSL2") - # X11 GUI socket can only be shared from a WSL (to find WSLg mount point) + # X11 socket can only be shared from a WSL (to find WSLg mount point) if EnvInfo.current_platform != "WSL": logger.debug("Exegol is running from a Windows context (e.g. Powershell), a WSL instance must be found to share WSLg X11 socket") cls.__distro_name = cls.__find_wsl_distro() logger.debug(f"Set WSL Distro as: '{cls.__distro_name}'") - # If no WSL is found, propose to continue without GUI + # If no WSL is found, propose to continue without GUI (X11 sharing) if not cls.__distro_name and not Confirm( - "Do you want to continue [orange3]without[/orange3] GUI support ?", default=True): + "Do you want to continue [orange3]without[/orange3] X11 sharing (i.e. GUI support)?", default=True): raise KeyboardInterrupt else: logger.debug("Using current WSL context for X11 socket sharing") @@ -280,7 +280,7 @@ def __wslg_eligible() -> bool: os_version_raw, _, build_number_raw = EnvInfo.getWindowsRelease().split('.')[:3] except ValueError: logger.debug(f"Impossible to find the version of windows: '{EnvInfo.getWindowsRelease()}'") - logger.error("Exegol can't know if your [orange3]version of Windows[/orange3] can support dockerized GUIs.") + logger.error("Exegol can't know if your [orange3]version of Windows[/orange3] can support dockerized GUIs (X11 sharing).") return False # Available from Windows 10 Build 21364 # Available from Windows 11 Build 22000 @@ -325,7 +325,7 @@ def __find_wsl_distro(cls) -> str: while not cls.__check_wsl_docker_integration(name): eligible = False logger.warning( - f"The '{name}' WSL distribution could be used to [green]enable the GUI[/green] on exegol but the docker integration is [orange3]not enabled[/orange3].") + f"The '{name}' WSL distribution can be used to [green]enable X11 sharing[/green] (i.e. GUI apps) on exegol but the docker integration is [orange3]not enabled[/orange3].") if not Confirm( f"Do you want to [red]manually[/red] enable docker integration for WSL '{name}'?", default=True): From 943e7488a40e9f67f2cb813c6730a706192f4468 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 10 Aug 2023 09:20:31 -0700 Subject: [PATCH 025/109] Internal service must be exposed to 0.0.0.0 wth port forwarding --- exegol/model/ContainerConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index fb1a3250..cf99df85 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -485,7 +485,7 @@ def enableDesktop(self, desktop_config: str = ""): self.addEnv(self.ExegolEnv.desktop_host.value, self.__desktop_host) self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__desktop_port)) else: - self.addEnv(self.ExegolEnv.desktop_host.value, "localhost") + self.addEnv(self.ExegolEnv.desktop_host.value, "0.0.0.0") self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__default_desktop_port.get(self.__desktop_proto))) # Exposing desktop service self.addPort(port_host=self.__desktop_port, port_container=self.__default_desktop_port[self.__desktop_proto], host_ip=self.__desktop_host) From aebe31e2d4bb61d06eb24152a988df78b0fa90a2 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Thu, 10 Aug 2023 09:25:07 -0700 Subject: [PATCH 026/109] Adding FIXME in desktop exposition --- exegol/model/ContainerConfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index cf99df85..f4b308c7 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -485,6 +485,7 @@ def enableDesktop(self, desktop_config: str = ""): self.addEnv(self.ExegolEnv.desktop_host.value, self.__desktop_host) self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__desktop_port)) else: + # FIXME : we are exposing the desktop on the network the container is in. Ex: if we're doing VPN, we're opening the desktop through it's network self.addEnv(self.ExegolEnv.desktop_host.value, "0.0.0.0") self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__default_desktop_port.get(self.__desktop_proto))) # Exposing desktop service From fbd9d6e780e126afdca6bb29ea9c5c94e454d855 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 22 Aug 2023 13:30:10 +0200 Subject: [PATCH 027/109] Fix warning message --- exegol/console/cli/actions/GenericParameters.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exegol/console/cli/actions/GenericParameters.py b/exegol/console/cli/actions/GenericParameters.py index cab30f6c..dd34597b 100644 --- a/exegol/console/cli/actions/GenericParameters.py +++ b/exegol/console/cli/actions/GenericParameters.py @@ -267,6 +267,7 @@ def __init__(self, groupArgs: List[GroupArg]): self.desktop = Option("--desktop", dest="desktop", action="store_true", + default=False, help=f"Enable or disable the Exegol desktop feature (default: {'[green]Enabled[/green]' if UserConfig().desktop_default_enable else '[red]Disabled[/red]'})") self.desktop_config = Option("--desktop-config", dest="desktop_config", From fb1a0ebdd5835f1382979e9ae2e257972a9b2915 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 24 Aug 2023 18:13:53 +0200 Subject: [PATCH 028/109] Add start.sh update procedure --- .github/workflows/sub_testing.yml | 2 + exegol/model/ContainerConfig.py | 14 +++- exegol/model/ExegolContainer.py | 26 +++++-- exegol/utils/imgsync/ImageScriptSync.py | 90 +++++++++++++++---------- exegol/utils/imgsync/start.sh | 6 ++ 5 files changed, 96 insertions(+), 42 deletions(-) mode change 100644 => 100755 exegol/utils/imgsync/start.sh diff --git a/.github/workflows/sub_testing.yml b/.github/workflows/sub_testing.yml index b7ed4829..ea43b16a 100644 --- a/.github/workflows/sub_testing.yml +++ b/.github/workflows/sub_testing.yml @@ -19,6 +19,8 @@ jobs: run: python -m pip install --user mypy types-requests types-PyYAML - name: Run code analysis run: mypy ./exegol/ --ignore-missing-imports --check-untyped-defs --pretty # TODO add --disallow-untyped-defs + - name: Find start.sh script version + run: egrep '^# Start Version:[0-9]+$' ./exegol/utils/imgsync/start.sh | cut -d ':' -f2 compatibility: name: Compatibility checks diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index f4b308c7..e2c512b7 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -12,6 +12,7 @@ from docker.types import Mount from rich.prompt import Prompt +from exegol import ConstantConfig from exegol.config.EnvInfo import EnvInfo from exegol.config.UserConfig import UserConfig from exegol.console.ConsoleFormat import boolFormatter, getColor @@ -70,6 +71,7 @@ def __init__(self, container: Optional[Container] = None): self.__exegol_resources: bool = False self.__network_host: bool = True self.__privileged: bool = False + self.__wrapper_start_enabled: bool = False self.__mounts: List[Mount] = [] self.__devices: List[str] = [] self.__capabilities: List[str] = [] @@ -101,6 +103,9 @@ def __init__(self, container: Optional[Container] = None): if container is not None: self.__parseContainerConfig(container) + else: + self.__wrapper_start_enabled = True + self.addVolume(str(ConstantConfig.start_context_path_obj), "/.exegol/start.sh", read_only=True, must_exist=True) # ===== Config parsing section ===== @@ -225,6 +230,8 @@ def __parseMounts(self, mounts: Optional[List[Dict]], name: str): obj_path = cast(PurePath, src_path) self.__vpn_path = obj_path logger.debug(f"Loading VPN config: {self.__vpn_path.name}") + elif "/.exegol/start.sh" in share.get('Destination', ''): + self.__wrapper_start_enabled = True # ===== Feature section ===== @@ -1079,6 +1086,10 @@ def getLabels(self) -> Dict[str, str]: self.addLabel(label_name, data) return self.__labels + def isWrapperStartShared(self) -> bool: + """Return True if the /.exegol/start.sh is a volume from the up-to-date wrapper script.""" + return self.__wrapper_start_enabled + # ===== Metadata labels getter / setter section ===== def setCreationDate(self, creation_date: str): @@ -1264,7 +1275,8 @@ def getTextMounts(self, verbose: bool = False) -> str: for mount in self.__mounts: # Blacklist technical mount if not verbose and mount.get('Target') in ['/tmp/.X11-unix', '/opt/resources', '/etc/localtime', - '/etc/timezone', '/my-resources', '/opt/my-resources']: + '/etc/timezone', '/my-resources', '/opt/my-resources', + '/.exegol/entrypoint.sh', '/.exegol/start.sh']: continue result += f"{mount.get('Source')} :right_arrow: {mount.get('Target')} {'(RO)' if mount.get('ReadOnly') else ''}{os.linesep}" return result diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 5e6b8205..ca0bea57 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -15,7 +15,7 @@ from exegol.model.SelectableInterface import SelectableInterface from exegol.utils.ContainerLogStream import ContainerLogStream from exegol.utils.ExeLog import logger, console -from exegol.utils.imgsync.ImageScriptSync import getImageSyncTarData +from exegol.utils.imgsync.ImageScriptSync import ImageScriptSync class ExegolContainer(ExegolContainerTemplate, SelectableInterface): @@ -104,7 +104,7 @@ def start(self): """Start the docker container""" if not self.isRunning(): logger.info(f"Starting container {self.name}") - self.preStartSetup() + self.__preStartSetup() self.__start_container() def __start_container(self): @@ -143,6 +143,7 @@ def stop(self, timeout: int = 10): def spawnShell(self): """Spawn a shell on the docker container""" + self.__check_start_version() logger.info(f"Location of the exegol workspace on the host : {self.config.getHostWorkspacePath()}") for device in self.config.getDevices(): logger.info(f"Shared host device: {device.split(':')[0]}") @@ -199,7 +200,7 @@ def formatShellCommand(command: Union[str, Sequence[str]], quiet: bool = False, - The second return argument is the command itself in str format.""" # Using base64 to escape special characters str_cmd = command if type(command) is str else ' '.join(command) - #str_cmd = str_cmd.replace('"', '\\"') # This fix shoudn' be necessary plus it can alter data like passwd + # str_cmd = str_cmd.replace('"', '\\"') # This fix shoudn' be necessary plus it can alter data like passwd if not quiet: logger.success(f"Command received: {str_cmd}") # ZSH pre-routine: Load zsh aliases and call eval to force aliases interpretation @@ -268,13 +269,28 @@ def __removeVolume(self): return logger.success("Private workspace volume removed successfully") - def preStartSetup(self): + def __preStartSetup(self): """ Operation to be performed before starting a container :return: """ self.__applyXhostACL() + def __check_start_version(self): + """ + Check start.sh up-to-date status and update the script if needed + :return: + """ + # Up-to-date container have the script shared over a volume + # But legacy container must be checked and the code must be pushed + if not self.config.isWrapperStartShared(): + # If the start.sh if not shared, the version must be compared and the script updated + current_start = ImageScriptSync.getCurrentStartVersion() + container_version = self.__container.exec_run(["/bin/bash", "-c", "egrep '^# Start Version:[0-9]+$' /.exegol/start.sh 2&>/dev/null || echo ':0' | cut -d ':' -f2"]).output.decode("utf-8").strip() + if current_start != container_version: + logger.debug(f"Updating start.sh script from version {container_version} to version {current_start}") + self.__container.put_archive("/", ImageScriptSync.getImageSyncTarData(include_start=True)) + def postCreateSetup(self, is_temporary: bool = False): """ Operation to be performed after creating a container @@ -284,7 +300,7 @@ def postCreateSetup(self, is_temporary: bool = False): # if not a temporary container, apply custom config if not is_temporary: # Update entrypoint script in the container - self.__container.put_archive("/", getImageSyncTarData()) + self.__container.put_archive("/", ImageScriptSync.getImageSyncTarData(include_entrypoint=True)) if self.__container.status.lower() == "created": self.__start_container() self.__updatePasswd() diff --git a/exegol/utils/imgsync/ImageScriptSync.py b/exegol/utils/imgsync/ImageScriptSync.py index bb603ebd..08de3b71 100644 --- a/exegol/utils/imgsync/ImageScriptSync.py +++ b/exegol/utils/imgsync/ImageScriptSync.py @@ -5,39 +5,57 @@ from exegol.utils.ExeLog import logger -def getImageSyncTarData(): - """The purpose of this class is to generate and overwrite scripts like the entrypoint or start.sh inside exegol containers - to integrate the latest features, whatever the version of the image.""" - - entrypoint_script_path = ConstantConfig.entrypoint_context_path_obj - logger.debug(f"Entrypoint script path: {str(entrypoint_script_path)}") - start_script_path = ConstantConfig.start_context_path_obj - logger.debug(f"Start script path: {str(start_script_path)}") - if not entrypoint_script_path.is_file() or not start_script_path.is_file(): - logger.error("Unable to find the entrypoint or start script! Your Exegol installation is probably broken...") - return None - # Create tar file - stream = io.BytesIO() - with tarfile.open(fileobj=stream, mode='w|') as entry_tar: - # Load entrypoint data - with open(entrypoint_script_path, 'rb') as f: - raw = f.read() - data = io.BytesIO(initial_bytes=raw) - - # Import file to tar object - info = tarfile.TarInfo(name="/.exegol/entrypoint.sh") - info.size = len(raw) - info.mode = 0o500 - entry_tar.addfile(info, fileobj=data) - - # Load start data - with open(start_script_path, 'rb') as f: - raw = f.read() - data = io.BytesIO(initial_bytes=raw) - - # Import file to tar object - info = tarfile.TarInfo(name="/.exegol/start.sh") - info.size = len(raw) - info.mode = 0o500 - entry_tar.addfile(info, fileobj=data) - return stream.getvalue() +class ImageScriptSync: + + @staticmethod + def getCurrentStartVersion(): + """Find the current version of the start.sh script.""" + with open(ConstantConfig.start_context_path_obj, 'r') as file: + for line in file.readlines(): + if line.startswith('# Start Version:'): + return line.split(':')[-1].strip() + logger.critical(f"The start.sh version cannot be found, check your exegol setup! {ConstantConfig.start_context_path_obj}") + + @staticmethod + def getImageSyncTarData(include_entrypoint: bool = False, include_start: bool = False): + """The purpose of this class is to generate and overwrite scripts like the entrypoint or start.sh inside exegol containers + to integrate the latest features, whatever the version of the image.""" + + # Create tar file + stream = io.BytesIO() + with tarfile.open(fileobj=stream, mode='w|') as entry_tar: + + # Load entrypoint data + if include_entrypoint: + entrypoint_script_path = ConstantConfig.entrypoint_context_path_obj + logger.debug(f"Entrypoint script path: {str(entrypoint_script_path)}") + if not entrypoint_script_path.is_file(): + logger.error("Unable to find the entrypoint script! Your Exegol installation is probably broken...") + return None + with open(entrypoint_script_path, 'rb') as f: + raw = f.read() + data = io.BytesIO(initial_bytes=raw) + + # Import file to tar object + info = tarfile.TarInfo(name="/.exegol/entrypoint.sh") + info.size = len(raw) + info.mode = 0o500 + entry_tar.addfile(info, fileobj=data) + + # Load start data + if include_start: + start_script_path = ConstantConfig.start_context_path_obj + logger.debug(f"Start script path: {str(start_script_path)}") + if not start_script_path.is_file(): + logger.error("Unable to find the start script! Your Exegol installation is probably broken...") + return None + with open(start_script_path, 'rb') as f: + raw = f.read() + data = io.BytesIO(initial_bytes=raw) + + # Import file to tar object + info = tarfile.TarInfo(name="/.exegol/start.sh") + info.size = len(raw) + info.mode = 0o500 + entry_tar.addfile(info, fileobj=data) + return stream.getvalue() diff --git a/exegol/utils/imgsync/start.sh b/exegol/utils/imgsync/start.sh old mode 100644 new mode 100755 index f1eb0c7d..3d22f1e9 --- a/exegol/utils/imgsync/start.sh +++ b/exegol/utils/imgsync/start.sh @@ -1,5 +1,11 @@ #!/bin/bash +# DO NOT CHANGE the syntax or text of the following line, only increment the version number +# Start Version:2 +# The start version allow the wrapper to compare the current version of the start.sh inside the container compare to the one on the current wrapper version. +# On new container, this file is automatically updated through a docker volume +# For legacy container, this version is fetch and the file updated if needed. + function shell_logging() { # First parameter is the method to use for shell logging (default to script) method=$1 From 2165efb7f0287a03d8cb027fda82cae21bd6f443 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 24 Aug 2023 18:14:38 +0200 Subject: [PATCH 029/109] Change chmod --- exegol/utils/imgsync/entrypoint.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 exegol/utils/imgsync/entrypoint.sh diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh old mode 100644 new mode 100755 From 53ad6ae28e4ac958d194a40e66e91f22f5077645 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 24 Aug 2023 18:18:42 +0200 Subject: [PATCH 030/109] Fix ConstantConfig import path --- exegol/model/ContainerConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index e2c512b7..e6d102ee 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -12,9 +12,9 @@ from docker.types import Mount from rich.prompt import Prompt -from exegol import ConstantConfig from exegol.config.EnvInfo import EnvInfo from exegol.config.UserConfig import UserConfig +from exegol.config.ConstantConfig import ConstantConfig from exegol.console.ConsoleFormat import boolFormatter, getColor from exegol.console.ExegolPrompt import Confirm from exegol.console.cli.ParametersManager import ParametersManager From 51ee2a0ac45739f28a6efa04a63959c1c520bfc6 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 24 Aug 2023 18:20:06 +0200 Subject: [PATCH 031/109] Add mypy source test --- .github/workflows/sub_testing.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sub_testing.yml b/.github/workflows/sub_testing.yml index ea43b16a..d4422250 100644 --- a/.github/workflows/sub_testing.yml +++ b/.github/workflows/sub_testing.yml @@ -17,8 +17,10 @@ jobs: python-version: "3.10" - name: Install requirements run: python -m pip install --user mypy types-requests types-PyYAML - - name: Run code analysis + - name: Run code analysis (package) run: mypy ./exegol/ --ignore-missing-imports --check-untyped-defs --pretty # TODO add --disallow-untyped-defs + - name: Run code analysis (source) + run: mypy ./exegol.py --ignore-missing-imports --check-untyped-defs --pretty # TODO add --disallow-untyped-defs - name: Find start.sh script version run: egrep '^# Start Version:[0-9]+$' ./exegol/utils/imgsync/start.sh | cut -d ':' -f2 From 1b21bc771fedf5fe80d9fd3b786e0a3f93527f24 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Fri, 25 Aug 2023 16:29:08 +0200 Subject: [PATCH 032/109] Disable start volume + support beta start versioning --- .github/workflows/entrypoint_prerelease.yml | 16 +++++++++++++++- .github/workflows/sub_testing.yml | 2 +- exegol/model/ContainerConfig.py | 6 +++--- exegol/utils/imgsync/start.sh | 2 +- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/entrypoint_prerelease.yml b/.github/workflows/entrypoint_prerelease.yml index 81ce57c2..c3b5d333 100644 --- a/.github/workflows/entrypoint_prerelease.yml +++ b/.github/workflows/entrypoint_prerelease.yml @@ -9,10 +9,24 @@ on: - "**.md" jobs: - test: + code_test: name: Python tests and checks uses: ./.github/workflows/sub_testing.yml + preprod_test: + name: Code testing + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@master + with: + submodules: false + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Find start.sh script version + run: egrep '^# Start Version:[0-9]+$' ./exegol/utils/imgsync/start.sh | cut -d ':' -f2 + build: name: Build Python 🐍 distributions runs-on: ubuntu-latest diff --git a/.github/workflows/sub_testing.yml b/.github/workflows/sub_testing.yml index d4422250..302a4cd4 100644 --- a/.github/workflows/sub_testing.yml +++ b/.github/workflows/sub_testing.yml @@ -22,7 +22,7 @@ jobs: - name: Run code analysis (source) run: mypy ./exegol.py --ignore-missing-imports --check-untyped-defs --pretty # TODO add --disallow-untyped-defs - name: Find start.sh script version - run: egrep '^# Start Version:[0-9]+$' ./exegol/utils/imgsync/start.sh | cut -d ':' -f2 + run: egrep '^# Start Version:[0-9ab]+$' ./exegol/utils/imgsync/start.sh | cut -d ':' -f2 compatibility: name: Compatibility checks diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index e6d102ee..842a2314 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -103,9 +103,9 @@ def __init__(self, container: Optional[Container] = None): if container is not None: self.__parseContainerConfig(container) - else: - self.__wrapper_start_enabled = True - self.addVolume(str(ConstantConfig.start_context_path_obj), "/.exegol/start.sh", read_only=True, must_exist=True) + #else: + # self.__wrapper_start_enabled = True + # self.addVolume(str(ConstantConfig.start_context_path_obj), "/.exegol/start.sh", read_only=True, must_exist=True) # ===== Config parsing section ===== diff --git a/exegol/utils/imgsync/start.sh b/exegol/utils/imgsync/start.sh index 3d22f1e9..f619c1ae 100755 --- a/exegol/utils/imgsync/start.sh +++ b/exegol/utils/imgsync/start.sh @@ -1,7 +1,7 @@ #!/bin/bash # DO NOT CHANGE the syntax or text of the following line, only increment the version number -# Start Version:2 +# Start Version:2b # The start version allow the wrapper to compare the current version of the start.sh inside the container compare to the one on the current wrapper version. # On new container, this file is automatically updated through a docker volume # For legacy container, this version is fetch and the file updated if needed. From e15c4c868703225439c0ed0cab770cf3f17b3713 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Mon, 28 Aug 2023 14:04:33 +0200 Subject: [PATCH 033/109] Rename start.sh to spawn.sh + warning for mac /opt path --- .github/workflows/entrypoint_prerelease.yml | 4 ++-- .github/workflows/sub_testing.yml | 4 ++-- exegol/config/ConstantConfig.py | 8 +++---- exegol/model/ContainerConfig.py | 21 ++++++++++------- exegol/model/ExegolContainer.py | 10 ++++---- exegol/utils/imgsync/ImageScriptSync.py | 26 ++++++++++----------- exegol/utils/imgsync/entrypoint.sh | 4 ++-- exegol/utils/imgsync/{start.sh => spawn.sh} | 4 ++-- setup.py | 2 +- 9 files changed, 44 insertions(+), 39 deletions(-) rename exegol/utils/imgsync/{start.sh => spawn.sh} (93%) diff --git a/.github/workflows/entrypoint_prerelease.yml b/.github/workflows/entrypoint_prerelease.yml index c3b5d333..3bfd1c7c 100644 --- a/.github/workflows/entrypoint_prerelease.yml +++ b/.github/workflows/entrypoint_prerelease.yml @@ -24,8 +24,8 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.10" - - name: Find start.sh script version - run: egrep '^# Start Version:[0-9]+$' ./exegol/utils/imgsync/start.sh | cut -d ':' -f2 + - name: Find spawn.sh script version + run: egrep '^# Spawn Version:[0-9]+$' ./exegol/utils/imgsync/spawn.sh | cut -d ':' -f2 build: name: Build Python 🐍 distributions diff --git a/.github/workflows/sub_testing.yml b/.github/workflows/sub_testing.yml index 302a4cd4..56e678ac 100644 --- a/.github/workflows/sub_testing.yml +++ b/.github/workflows/sub_testing.yml @@ -21,8 +21,8 @@ jobs: run: mypy ./exegol/ --ignore-missing-imports --check-untyped-defs --pretty # TODO add --disallow-untyped-defs - name: Run code analysis (source) run: mypy ./exegol.py --ignore-missing-imports --check-untyped-defs --pretty # TODO add --disallow-untyped-defs - - name: Find start.sh script version - run: egrep '^# Start Version:[0-9ab]+$' ./exegol/utils/imgsync/start.sh | cut -d ':' -f2 + - name: Find spawn.sh script version + run: egrep '^# Spawn Version:[0-9ab]+$' ./exegol/utils/imgsync/spawn.sh | cut -d ':' -f2 compatibility: name: Compatibility checks diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index edb37c03..2103c8eb 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -15,10 +15,10 @@ class ConstantConfig: # Path of the Dockerfile build_context_path_obj: Path build_context_path: str - # Path of the Entrypoint.sh + # Path of the entrypoint.sh entrypoint_context_path_obj: Path - # Path of the Start.sh - start_context_path_obj: Path + # Path of the spawn.sh + spawn_context_path_obj: Path # Exegol config directory exegol_config_path: Path = Path().home() / ".exegol" # Docker Desktop for mac config file @@ -66,4 +66,4 @@ def findResourceContextPath(cls, resource_folder: str, source_path: str) -> Path ConstantConfig.build_context_path = str(ConstantConfig.build_context_path_obj) ConstantConfig.entrypoint_context_path_obj = ConstantConfig.findResourceContextPath("exegol-imgsync", "exegol/utils/imgsync/entrypoint.sh") -ConstantConfig.start_context_path_obj = ConstantConfig.findResourceContextPath("exegol-imgsync", "exegol/utils/imgsync/start.sh") +ConstantConfig.spawn_context_path_obj = ConstantConfig.findResourceContextPath("exegol-imgsync", "exegol/utils/imgsync/spawn.sh") diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 842a2314..22089875 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -103,9 +103,9 @@ def __init__(self, container: Optional[Container] = None): if container is not None: self.__parseContainerConfig(container) - #else: - # self.__wrapper_start_enabled = True - # self.addVolume(str(ConstantConfig.start_context_path_obj), "/.exegol/start.sh", read_only=True, must_exist=True) + else: + self.__wrapper_start_enabled = True + self.addVolume(str(ConstantConfig.spawn_context_path_obj), "/.exegol/spawn.sh", read_only=True, must_exist=True) # ===== Config parsing section ===== @@ -230,7 +230,7 @@ def __parseMounts(self, mounts: Optional[List[Dict]], name: str): obj_path = cast(PurePath, src_path) self.__vpn_path = obj_path logger.debug(f"Loading VPN config: {self.__vpn_path.name}") - elif "/.exegol/start.sh" in share.get('Destination', ''): + elif "/.exegol/spawn.sh" in share.get('Destination', ''): self.__wrapper_start_enabled = True # ===== Feature section ===== @@ -730,8 +730,8 @@ def getEntrypointCommand(self) -> Tuple[Optional[List[str]], Union[List[str], st def getShellCommand(self) -> str: """Get container command for opening a new shell""" - # Use a start.sh script to handle features with the wrapper - return "/.exegol/start.sh" + # Use a spawn.sh script to handle features with the wrapper + return "/.exegol/spawn.sh" @staticmethod def generateRandomPassword(length: int = 30) -> str: @@ -902,6 +902,11 @@ def addVolume(self, if EnvInfo.isMacHost(): # Add support for /etc path_match = str(path) + if path_match.startswith("/opt/"): + msg = f"{EnvInfo.getDockerEngine()} cannot mount directory from [magenta]/opt/[/magenta] host path." + if path_match.endswith("entrypoint.sh") or path_match.endswith("spawn.sh"): + msg += " Your exegol installation cannot be stored under this directory." + raise CancelOperation(f"{EnvInfo.getDockerEngine()} cannot mount directory from [magenta]/opt/[/magenta] host path.") if path_match.startswith("/etc/"): if EnvInfo.isOrbstack(): raise CancelOperation(f"Orbstack doesn't support sharing /etc files with the container") @@ -1087,7 +1092,7 @@ def getLabels(self) -> Dict[str, str]: return self.__labels def isWrapperStartShared(self) -> bool: - """Return True if the /.exegol/start.sh is a volume from the up-to-date wrapper script.""" + """Return True if the /.exegol/spawn.sh is a volume from the up-to-date wrapper script.""" return self.__wrapper_start_enabled # ===== Metadata labels getter / setter section ===== @@ -1276,7 +1281,7 @@ def getTextMounts(self, verbose: bool = False) -> str: # Blacklist technical mount if not verbose and mount.get('Target') in ['/tmp/.X11-unix', '/opt/resources', '/etc/localtime', '/etc/timezone', '/my-resources', '/opt/my-resources', - '/.exegol/entrypoint.sh', '/.exegol/start.sh']: + '/.exegol/entrypoint.sh', '/.exegol/spawn.sh']: continue result += f"{mount.get('Source')} :right_arrow: {mount.get('Target')} {'(RO)' if mount.get('ReadOnly') else ''}{os.linesep}" return result diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index ca0bea57..2fb2a1cf 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -278,18 +278,18 @@ def __preStartSetup(self): def __check_start_version(self): """ - Check start.sh up-to-date status and update the script if needed + Check spawn.sh up-to-date status and update the script if needed :return: """ # Up-to-date container have the script shared over a volume # But legacy container must be checked and the code must be pushed if not self.config.isWrapperStartShared(): - # If the start.sh if not shared, the version must be compared and the script updated + # If the spawn.sh if not shared, the version must be compared and the script updated current_start = ImageScriptSync.getCurrentStartVersion() - container_version = self.__container.exec_run(["/bin/bash", "-c", "egrep '^# Start Version:[0-9]+$' /.exegol/start.sh 2&>/dev/null || echo ':0' | cut -d ':' -f2"]).output.decode("utf-8").strip() + container_version = self.__container.exec_run(["/bin/bash", "-c", "egrep '^# Spawn Version:[0-9]+$' /.exegol/spawn.sh 2&>/dev/null || echo ':0' | cut -d ':' -f2"]).output.decode("utf-8").strip() if current_start != container_version: - logger.debug(f"Updating start.sh script from version {container_version} to version {current_start}") - self.__container.put_archive("/", ImageScriptSync.getImageSyncTarData(include_start=True)) + logger.debug(f"Updating spawn.sh script from version {container_version} to version {current_start}") + self.__container.put_archive("/", ImageScriptSync.getImageSyncTarData(include_spawn=True)) def postCreateSetup(self, is_temporary: bool = False): """ diff --git a/exegol/utils/imgsync/ImageScriptSync.py b/exegol/utils/imgsync/ImageScriptSync.py index 08de3b71..36540609 100644 --- a/exegol/utils/imgsync/ImageScriptSync.py +++ b/exegol/utils/imgsync/ImageScriptSync.py @@ -9,16 +9,16 @@ class ImageScriptSync: @staticmethod def getCurrentStartVersion(): - """Find the current version of the start.sh script.""" - with open(ConstantConfig.start_context_path_obj, 'r') as file: + """Find the current version of the spawn.sh script.""" + with open(ConstantConfig.spawn_context_path_obj, 'r') as file: for line in file.readlines(): - if line.startswith('# Start Version:'): + if line.startswith('# Spawn Version:'): return line.split(':')[-1].strip() - logger.critical(f"The start.sh version cannot be found, check your exegol setup! {ConstantConfig.start_context_path_obj}") + logger.critical(f"The spawn.sh version cannot be found, check your exegol setup! {ConstantConfig.spawn_context_path_obj}") @staticmethod - def getImageSyncTarData(include_entrypoint: bool = False, include_start: bool = False): - """The purpose of this class is to generate and overwrite scripts like the entrypoint or start.sh inside exegol containers + def getImageSyncTarData(include_entrypoint: bool = False, include_spawn: bool = False): + """The purpose of this class is to generate and overwrite scripts like the entrypoint or spawn.sh inside exegol containers to integrate the latest features, whatever the version of the image.""" # Create tar file @@ -43,18 +43,18 @@ def getImageSyncTarData(include_entrypoint: bool = False, include_start: bool = entry_tar.addfile(info, fileobj=data) # Load start data - if include_start: - start_script_path = ConstantConfig.start_context_path_obj - logger.debug(f"Start script path: {str(start_script_path)}") - if not start_script_path.is_file(): - logger.error("Unable to find the start script! Your Exegol installation is probably broken...") + if include_spawn: + spawn_script_path = ConstantConfig.spawn_context_path_obj + logger.debug(f"Spawn script path: {str(spawn_script_path)}") + if not spawn_script_path.is_file(): + logger.error("Unable to find the spawn script! Your Exegol installation is probably broken...") return None - with open(start_script_path, 'rb') as f: + with open(spawn_script_path, 'rb') as f: raw = f.read() data = io.BytesIO(initial_bytes=raw) # Import file to tar object - info = tarfile.TarInfo(name="/.exegol/start.sh") + info = tarfile.TarInfo(name="/.exegol/spawn.sh") info.size = len(raw) info.mode = 0o500 entry_tar.addfile(info, fileobj=data) diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh index 12caf2a9..3b390875 100755 --- a/exegol/utils/imgsync/entrypoint.sh +++ b/exegol/utils/imgsync/entrypoint.sh @@ -3,7 +3,7 @@ trap shutdown SIGTERM function exegol_init() { - usermod -s "/.exegol/start.sh" root + usermod -s "/.exegol/spawn.sh" root } # Function specific @@ -52,7 +52,7 @@ function shutdown() { # shellcheck disable=SC2046 kill $(pgrep -x -f -- -bash) 2>/dev/null # Wait for every active process to exit (e.g: shell logging compression, VPN closing, WebUI) - wait_list="$(pgrep -f "(.log|start.sh|vnc)" | grep -vE '^1$')" + wait_list="$(pgrep -f "(.log|spawn.sh|vnc)" | grep -vE '^1$')" for i in $wait_list; do # Waiting for: $i PID process to exit tail --pid="$i" -f /dev/null diff --git a/exegol/utils/imgsync/start.sh b/exegol/utils/imgsync/spawn.sh similarity index 93% rename from exegol/utils/imgsync/start.sh rename to exegol/utils/imgsync/spawn.sh index f619c1ae..0802de92 100755 --- a/exegol/utils/imgsync/start.sh +++ b/exegol/utils/imgsync/spawn.sh @@ -1,8 +1,8 @@ #!/bin/bash # DO NOT CHANGE the syntax or text of the following line, only increment the version number -# Start Version:2b -# The start version allow the wrapper to compare the current version of the start.sh inside the container compare to the one on the current wrapper version. +# Spawn Version:2b +# The spawn version allow the wrapper to compare the current version of the spawn.sh inside the container compare to the one on the current wrapper version. # On new container, this file is automatically updated through a docker volume # For legacy container, this version is fetch and the file updated if needed. diff --git a/setup.py b/setup.py index 7c438d83..eedac5ff 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ data_files_dict[key].append(str(path)) ## exegol scripts pushed from the wrapper data_files_dict["exegol-imgsync"] = ["exegol/utils/imgsync/entrypoint.sh"] -data_files_dict["exegol-imgsync"] = ["exegol/utils/imgsync/start.sh"] +data_files_dict["exegol-imgsync"] = ["exegol/utils/imgsync/spawn.sh"] # Dict to tuple for k, v in data_files_dict.items(): From e699157e3574ac82d37d6953fbea68ac8de2e012 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Mon, 28 Aug 2023 14:44:30 +0200 Subject: [PATCH 034/109] Adding ro|rw to start help dialog + debug info tables --- exegol/console/cli/actions/ExegolParameters.py | 5 +++-- exegol/console/cli/actions/GenericParameters.py | 6 +++--- exegol/model/ContainerConfig.py | 15 +++++++++------ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/exegol/console/cli/actions/ExegolParameters.py b/exegol/console/cli/actions/ExegolParameters.py index 8ba3b9df..3ccb7712 100644 --- a/exegol/console/cli/actions/ExegolParameters.py +++ b/exegol/console/cli/actions/ExegolParameters.py @@ -21,9 +21,10 @@ def __init__(self): "Create a container [blue]test[/blue] with a custom shared workspace": "exegol start [blue]test[/blue] [bright_blue]full[/bright_blue] -w [magenta]./project/pentest/[/magenta]", "Create a container [blue]test[/blue] sharing the current working directory": "exegol start [blue]test[/blue] [bright_blue]full[/bright_blue] -cwd", "Create a container [blue]htb[/blue] with a VPN": "exegol start [blue]htb[/blue] [bright_blue]full[/bright_blue] --vpn [magenta]~/vpn/[/magenta][bright_magenta]lab_Dramelac.ovpn[/bright_magenta]", - "Create a container [blue]app[/blue] with custom volume": "exegol start [blue]app[/blue] [bright_blue]full[/bright_blue] -V [bright_magenta]'/var/app/:/app/'[/bright_magenta]", + "Create a container [blue]app[/blue] with custom volume": "exegol start [blue]app[/blue] [bright_blue]full[/bright_blue] -V [bright_magenta]/var/app/[/bright_magenta]:[bright_magenta]/app/[/bright_magenta]", + "Create a container [blue]app[/blue] with custom volume in [blue]ReadOnly[/blue]": "exegol start [blue]app[/blue] [bright_blue]full[/bright_blue] -V [bright_magenta]/var/app/[/bright_magenta]:[bright_magenta]/app/[/bright_magenta]:[blue]ro[/blue]", "Get a [blue]tmux[/blue] shell": "exegol start --shell [blue]tmux[/blue]", - "Share a specific [blue]hardware device[/blue] (like Proxmark)": "exegol start -d /dev/ttyACM0", + "Share a specific [blue]hardware device[/blue] [bright_black](e.g. Proxmark)[/bright_black]": "exegol start -d /dev/ttyACM0", "Share every [blue]USB device[/blue] connected to the host": "exegol start -d /dev/bus/usb/", } diff --git a/exegol/console/cli/actions/GenericParameters.py b/exegol/console/cli/actions/GenericParameters.py index dd34597b..30dedca0 100644 --- a/exegol/console/cli/actions/GenericParameters.py +++ b/exegol/console/cli/actions/GenericParameters.py @@ -191,7 +191,7 @@ def __init__(self, groupArgs: List[GroupArg]): action="append", default=[], dest="volumes", - help="Share a new volume between host and exegol (format: --volume /path/on/host/:/path/in/container/)") + help="Share a new volume between host and exegol (format: --volume /path/on/host/:/path/in/container/[blue][:ro|rw][/blue])") self.ports = Option("-p", "--port", action="append", default=[], @@ -273,8 +273,8 @@ def __init__(self, groupArgs: List[GroupArg]): dest="desktop_config", default="", action="store", - help=f"Configure your exegol desktop ([blue]{', '.join(UserConfig.desktop_available_proto)}[/blue]) and its exposition " - f"(format: [blue]proto[/blue]\[:[blue]ip[/blue]\[:[blue]port[/blue]]]) " + help=f"Configure your exegol desktop ([blue]{'[/blue] or [blue]'.join(UserConfig.desktop_available_proto)}[/blue]) and its exposure " + f"(format: [blue]proto[:ip[:port]][/blue]) " f"(default: [blue]{UserConfig().desktop_default_proto}[/blue]:[blue]{'127.0.0.1' if UserConfig().desktop_default_localhost else '0.0.0.0'}[/blue]:[blue][/blue])", completer=DesktopConfigCompleter) groupArgs.append(GroupArg({"arg": self.desktop, "required": False}, diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 22089875..bbe09ed2 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -1160,7 +1160,7 @@ def addRawVolume(self, volume_string): logger.debug( f"Adding a volume from '{host_path}' to '{container_path}' as {'readonly' if readonly else 'read/write'}") try: - self.addVolume(host_path, container_path, readonly) + self.addVolume(host_path, container_path, read_only=readonly) except CancelOperation as e: logger.error(f"The following volume couldn't be created [magenta]{volume_string}[/magenta]. {e}") if not Confirm("Do you want to continue without this volume ?", False): @@ -1277,13 +1277,16 @@ def getTextCreationDate(self) -> str: def getTextMounts(self, verbose: bool = False) -> str: """Text formatter for Mounts configurations. The verbose mode does not exclude technical volumes.""" result = '' + hidden_mounts = ['/tmp/.X11-unix', '/opt/resources', '/etc/localtime', + '/etc/timezone', '/my-resources', '/opt/my-resources', + '/.exegol/entrypoint.sh', '/.exegol/spawn.sh'] for mount in self.__mounts: - # Blacklist technical mount - if not verbose and mount.get('Target') in ['/tmp/.X11-unix', '/opt/resources', '/etc/localtime', - '/etc/timezone', '/my-resources', '/opt/my-resources', - '/.exegol/entrypoint.sh', '/.exegol/spawn.sh']: + # Not showing technical mounts + if not verbose and mount.get('Target') in hidden_mounts: continue - result += f"{mount.get('Source')} :right_arrow: {mount.get('Target')} {'(RO)' if mount.get('ReadOnly') else ''}{os.linesep}" + read_only_text = f"[bright_black](RO)[/bright_black] " if verbose else '' + read_write_text = f"[orange3](RW)[/orange3] " if verbose else '' + result += f"{read_only_text if mount.get('ReadOnly') else read_write_text}{mount.get('Source')} :right_arrow: {mount.get('Target')}{os.linesep}" return result def getTextDevices(self, verbose: bool = False) -> str: From e131eac3b584be17e116c608a19f0b5aa43e07e9 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 29 Aug 2023 14:04:46 +0200 Subject: [PATCH 035/109] Fix enum print value --- exegol/manager/ExegolManager.py | 6 +++--- exegol/model/ContainerConfig.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index ec0d62d0..83e97224 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -212,16 +212,16 @@ def print_version(cls): logger.warning("You are currently using a [orange3]Beta[/orange3] version of Exegol, which may be unstable.") logger.debug(f"Pip installation: {boolFormatter(ConstantConfig.pip_installed)}") logger.debug(f"Git source installation: {boolFormatter(ConstantConfig.git_source_installation)}") - logger.debug(f"Host OS: {EnvInfo.getHostOs()} [bright_black]({EnvInfo.getDockerEngine()})[/bright_black]") + logger.debug(f"Host OS: {EnvInfo.getHostOs().value} [bright_black]({EnvInfo.getDockerEngine().value})[/bright_black]") logger.debug(f"Arch: {EnvInfo.arch}") if EnvInfo.arch != EnvInfo.raw_arch: logger.debug(f"Raw arch: {EnvInfo.raw_arch}") if EnvInfo.isWindowsHost(): logger.debug(f"Windows release: {EnvInfo.getWindowsRelease()}") logger.debug(f"Python environment: {EnvInfo.current_platform}") - logger.debug(f"Docker engine: {str(EnvInfo.getDockerEngine()).upper()}") + logger.debug(f"Docker engine: {EnvInfo.getDockerEngine().value}") logger.debug(f"Docker desktop: {boolFormatter(EnvInfo.isDockerDesktop())}") - logger.debug(f"Shell type: {EnvInfo.getShellType()}") + logger.debug(f"Shell type: {EnvInfo.getShellType().value}") if not UpdateManager.isUpdateTag() and UserConfig().auto_check_updates: UpdateManager.checkForWrapperUpdate() if UpdateManager.isUpdateTag(): diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index bbe09ed2..599b3bf8 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -903,10 +903,10 @@ def addVolume(self, # Add support for /etc path_match = str(path) if path_match.startswith("/opt/"): - msg = f"{EnvInfo.getDockerEngine()} cannot mount directory from [magenta]/opt/[/magenta] host path." + msg = f"{EnvInfo.getDockerEngine().value} cannot mount directory from [magenta]/opt/[/magenta] host path." if path_match.endswith("entrypoint.sh") or path_match.endswith("spawn.sh"): msg += " Your exegol installation cannot be stored under this directory." - raise CancelOperation(f"{EnvInfo.getDockerEngine()} cannot mount directory from [magenta]/opt/[/magenta] host path.") + raise CancelOperation(f"{EnvInfo.getDockerEngine().value} cannot mount directory from [magenta]/opt/[/magenta] host path.") if path_match.startswith("/etc/"): if EnvInfo.isOrbstack(): raise CancelOperation(f"Orbstack doesn't support sharing /etc files with the container") From d303809897002d0ff6d3293856afd12f327e27a0 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Tue, 29 Aug 2023 14:07:23 +0200 Subject: [PATCH 036/109] Fix error handling --- exegol/model/ContainerConfig.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 599b3bf8..b37ae183 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -906,7 +906,8 @@ def addVolume(self, msg = f"{EnvInfo.getDockerEngine().value} cannot mount directory from [magenta]/opt/[/magenta] host path." if path_match.endswith("entrypoint.sh") or path_match.endswith("spawn.sh"): msg += " Your exegol installation cannot be stored under this directory." - raise CancelOperation(f"{EnvInfo.getDockerEngine().value} cannot mount directory from [magenta]/opt/[/magenta] host path.") + logger.critical(msg) + raise CancelOperation(msg) if path_match.startswith("/etc/"): if EnvInfo.isOrbstack(): raise CancelOperation(f"Orbstack doesn't support sharing /etc files with the container") From 3af518c298e3fb0106d328fa8202715b6fa35d3d Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:42:18 +0200 Subject: [PATCH 037/109] adding logs for datacache update --- exegol/config/DataCache.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/exegol/config/DataCache.py b/exegol/config/DataCache.py index cee1309f..08d355d5 100644 --- a/exegol/config/DataCache.py +++ b/exegol/config/DataCache.py @@ -59,6 +59,21 @@ def get_images_data(self) -> ImagesCacheModel: def update_image_cache(self, images: List): """Refresh image cache data""" - cache_images = [ImageCacheModel(img.getName(), img.getLatestVersion(), img.getLatestRemoteId(), "local" if img.isLocal() else "remote") for img in images] + logger.debug("Updating image cache data") + cache_images = [] + for img in images: + name = img.getName() + version = img.getLatestVersion() + remoteid = img.getLatestRemoteId() + type = "local" if img.isLocal() else "remote" + logger.debug(f"└── {name} (version: {version})\t→ ({type}) {remoteid}") + cache_images.append( + ImageCacheModel( + name, + version, + remoteid, + type + ) + ) self.__cache_data.images = ImagesCacheModel(cache_images) self.save_updates() From 48080ad0efb6f34b590ad56709d0f187bc950535 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Wed, 30 Aug 2023 14:42:50 +0200 Subject: [PATCH 038/109] better looks for container and image logs --- exegol/model/ContainerConfig.py | 18 +++++++++--------- exegol/model/ExegolContainer.py | 2 +- exegol/model/ExegolImage.py | 7 ++++--- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index b37ae183..7b74d2b2 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -131,14 +131,14 @@ def __parseContainerConfig(self, container: Container): caps = host_config.get("CapAdd", []) if caps is not None: self.__capabilities = caps - logger.debug(f"Capabilities : {self.__capabilities}") + logger.debug(f"└── Capabilities : {self.__capabilities}") self.__sysctls = host_config.get("Sysctls", {}) devices = host_config.get("Devices", []) if devices is not None: for device in devices: self.__devices.append( f"{device.get('PathOnHost', '?')}:{device.get('PathInContainer', '?')}:{device.get('CgroupPermissions', '?')}") - logger.debug(f"Load devices : {self.__devices}") + logger.debug(f"└── Load devices : {self.__devices}") # Volumes section self.__share_timezone = False @@ -153,7 +153,7 @@ def __parseContainerConfig(self, container: Container): def __parseEnvs(self, envs: List[str]): """Parse envs object syntax""" for env in envs: - logger.debug(f"Parsing envs : {env}") + logger.debug(f"└── Parsing envs : {env}") # Removing " and ' at the beginning and the end of the string before splitting key / value self.addRawEnv(env.strip("'").strip('"')) @@ -162,7 +162,7 @@ def __parseLabels(self, labels: Dict[str, str]): for key, value in labels.items(): if not key.startswith("org.exegol."): continue - logger.debug(f"Parsing label : {key}") + logger.debug(f"└── Parsing label : {key}") if key.startswith("org.exegol.metadata."): # Find corresponding feature and attributes refs = self.__label_metadata.get(key) # Setter @@ -184,7 +184,7 @@ def __parseMounts(self, mounts: Optional[List[Dict]], name: str): mounts = [] self.__disable_workspace = True for share in mounts: - logger.debug(f"Parsing mount : {share}") + logger.debug(f"└── Parsing mount : {share}") src_path: Optional[PurePath] = None obj_path: PurePath if share.get('Type', 'volume') == "volume": @@ -214,14 +214,14 @@ def __parseMounts(self, mounts: Optional[List[Dict]], name: str): # Workspace are always bind mount assert src_path is not None obj_path = cast(PurePath, src_path) - logger.debug(f"Loading workspace volume source : {obj_path}") + logger.debug(f"└── Loading workspace volume source : {obj_path}") self.__disable_workspace = False if obj_path is not None and obj_path.name == name and \ (obj_path.parent.name == "shared-data-volumes" or obj_path.parent == UserConfig().private_volume_path): # Check legacy path and new custom path - logger.debug("Private workspace detected") + logger.debug("└── Private workspace detected") self.__workspace_dedicated_path = str(obj_path) else: - logger.debug("Custom workspace detected") + logger.debug("└── Custom workspace detected") self.__workspace_custom_path = str(obj_path) # TODO remove support for previous container elif "/vpn" in share.get('Destination', '') or "/.exegol/vpn" in share.get('Destination', ''): @@ -229,7 +229,7 @@ def __parseMounts(self, mounts: Optional[List[Dict]], name: str): assert src_path is not None obj_path = cast(PurePath, src_path) self.__vpn_path = obj_path - logger.debug(f"Loading VPN config: {self.__vpn_path.name}") + logger.debug(f"└── Loading VPN config: {self.__vpn_path.name}") elif "/.exegol/spawn.sh" in share.get('Destination', ''): self.__wrapper_start_enabled = True diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 2fb2a1cf..f3b40194 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -22,7 +22,7 @@ class ExegolContainer(ExegolContainerTemplate, SelectableInterface): """Class of an exegol container already create in docker""" def __init__(self, docker_container: Container, model: Optional[ExegolContainerTemplate] = None): - logger.debug(f"== Loading container : {docker_container.name}") + logger.debug(f"Loading container: {docker_container.name}") self.__container: Container = docker_container self.__id: str = docker_container.id self.__xhost_applied = False diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index ffa67df0..91218da2 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -216,7 +216,6 @@ def __labelVersionParsing(self): """Fallback version parsing using image's label (if exist). This method can only be used if version has not been provided from the image's tag.""" if "N/A" in self.__image_version and self.__image is not None: - logger.debug("Try to retrieve image version from labels") version_label = self.__image.labels.get("org.exegol.version") if version_label is not None: self.__setImageVersion(version_label, source_tag=False) @@ -354,6 +353,7 @@ def mergeImages(cls, remote_images: List[MetaImages], local_images: List[Image]) - unknown : no internet connection = no information from the registry - not install : other remote images without any match Return a list of ordered ExegolImage.""" + logger.debug("Comparing and merging local and remote images data") results = [] latest_installed: List[str] = [] cls.__mergeMetaImages(remote_images) @@ -362,7 +362,8 @@ def mergeImages(cls, remote_images: List[MetaImages], local_images: List[Image]) for r_img in remote_images: remote_img_dict[r_img.name] = r_img - # Find a match for each local image + # Searching a match for each local image + logger.debug("Searching a match for each image installed") for img in local_images: current_local_img = ExegolImage(docker_image=img) # quick handle of local images @@ -511,7 +512,7 @@ def getStatus(self, include_version: bool = True) -> str: if self.getLatestVersion(): status += f" (v.{self.getImageVersion()} :arrow_right: v.{self.getLatestVersion()})" else: - status += f" (v.{self.getImageVersion()})" + status += f" (currently v.{self.getImageVersion()})" status += "[/orange3]" return status else: From 09b79f9a0f0f9c5a9c2b732f7d0700d1ecd34f65 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Thu, 31 Aug 2023 02:14:43 +0200 Subject: [PATCH 039/109] changing downloads badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2a906778..6ed09072 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@

pip package version Python3.7 - pip stats + latest commit on master

latest commit on master latest commit on dev From 016b56b58c85e57571ceaa6c8bfa1ac991bd14fb Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sat, 2 Sep 2023 02:54:33 +0200 Subject: [PATCH 040/109] better logs for image builds --- exegol/console/TUI.py | 2 +- exegol/utils/DockerUtils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exegol/console/TUI.py b/exegol/console/TUI.py index 100b154c..848af017 100644 --- a/exegol/console/TUI.py +++ b/exegol/console/TUI.py @@ -145,7 +145,7 @@ def buildDockerImage(build_stream: Generator): else: logger.raw(stream_text, level=ExeLog.ADVANCED) if ': FROM ' in stream_text: - logger.info("Downloading docker image") + logger.info("Downloading base image") ExegolTUI.downloadDockerLayer(build_stream, quick_exit=True) if logfile is not None: logfile.close() diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index d1464fb7..4eab322d 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -536,7 +536,7 @@ def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerf if build_profile is None or build_dockerfile is None: build_profile = "full" build_dockerfile = "Dockerfile" - logger.info("Starting build. Please wait, this might be [bold](very)[/bold] long.") + logger.info("Starting build. Please wait, this will be long.") logger.verbose(f"Creating build context from [gold]{ConstantConfig.build_context_path}[/gold] with " f"[green][b]{build_profile}[/b][/green] profile ({ParametersManager().arch}).") if EnvInfo.arch != ParametersManager().arch: From 068b3e1857504fc01d28871b557cde8f7aebb7dd Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sat, 2 Sep 2023 02:54:52 +0200 Subject: [PATCH 041/109] adding note on workflows pr trigger paths-ignore filter --- .github/workflows/entrypoint_prerelease.yml | 2 +- .github/workflows/entrypoint_pull_request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/entrypoint_prerelease.yml b/.github/workflows/entrypoint_prerelease.yml index 3bfd1c7c..ad9127f6 100644 --- a/.github/workflows/entrypoint_prerelease.yml +++ b/.github/workflows/entrypoint_prerelease.yml @@ -4,7 +4,7 @@ on: pull_request: branches: - "master" - paths-ignore: + paths-ignore: # not always respected. See https://github.com/actions/runner/issues/2324#issuecomment-1703345084 - ".github/**" - "**.md" diff --git a/.github/workflows/entrypoint_pull_request.yml b/.github/workflows/entrypoint_pull_request.yml index de2b2fa4..59d06ea0 100644 --- a/.github/workflows/entrypoint_pull_request.yml +++ b/.github/workflows/entrypoint_pull_request.yml @@ -7,7 +7,7 @@ on: pull_request: branches-ignore: - "master" - paths-ignore: + paths-ignore: # not always respected. See https://github.com/actions/runner/issues/2324#issuecomment-1703345084 - ".github/**" - "**.md" push: From 35bc0d7f7b2868de7e2a43c9485a1fae38d1c72e Mon Sep 17 00:00:00 2001 From: Charlie Bromberg <40902872+ShutdownRepo@users.noreply.github.com> Date: Thu, 7 Sep 2023 14:56:13 +0200 Subject: [PATCH 042/109] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ed09072..6f156230 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ # Getting started -You can refer to the [Exegol documentations](https://exegol.readthedocs.io/en/latest/getting-started/install.html). +You can refer to the [Exegol documentation](https://exegol.readthedocs.io/en/latest/getting-started/install.html). > Full documentation homepage: https://exegol.rtfd.io/. From fe8df5a17ab9b894667ab443cc101f697ba1f9fb Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Wed, 27 Sep 2023 22:10:37 +0200 Subject: [PATCH 043/109] catching exception and returning port -1 when cannot bind --- exegol/model/ContainerConfig.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 7b74d2b2..f924ac10 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -744,10 +744,16 @@ def generateRandomPassword(length: int = 30) -> str: @staticmethod def __findAvailableRandomPort(interface: str = 'localhost') -> int: """Find an available random port. Using the socket system to """ + logger.debug(f"Attempting to bind to interface {interface}") import socket sock = socket.socket() - sock.bind((interface, 0)) # Using port 0 let the system decide for a random port + try: + sock.bind((interface, 0)) # Using port 0 let the system decide for a random port + except OSError as e: + logger.error(f"Unable to bind to interface/port: {e}") + return -1 random_port = sock.getsockname()[1] + logger.debug(f"Found available port {random_port}") sock.close() return random_port From 3b9df15decbb087aa12bbe983db2c045b9a4f461 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Wed, 27 Sep 2023 23:05:25 +0200 Subject: [PATCH 044/109] fixing port definition logic --- exegol/model/ContainerConfig.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index f924ac10..13099dda 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -504,20 +504,24 @@ def configureDesktop(self, desktop_config: str): """ self.__desktop_proto = UserConfig().desktop_default_proto self.__desktop_host = "127.0.0.1" if UserConfig().desktop_default_localhost else "0.0.0.0" + _host_set_by_user = False for i, data in enumerate(desktop_config.split(":")): if not data: continue - if i == 0: + if i == 0: # protocol + logger.debug(f"Desktop proto set: {data}") data = data.lower() if data in UserConfig.desktop_available_proto: self.__desktop_proto = data else: logger.critical(f"The desktop mode '{data}' is not supported. Please choose a supported mode: [green]{', '.join(UserConfig.desktop_available_proto)}[/green].") - elif i == 1 and data: + elif i == 1 and data: # host + logger.debug(f"Desktop host set: {data}") self.__desktop_host = data - self.__desktop_port = self.__findAvailableRandomPort(self.__desktop_host) - elif i == 2: + _host_set_by_user = True + elif i == 2: # port + logger.debug(f"Desktop port set: {data}") try: self.__desktop_port = int(data) except ValueError: @@ -526,7 +530,9 @@ def configureDesktop(self, desktop_config: str): logger.critical(f"Your configuration is invalid, please use the following format:[green]mode:host:port[/green]") if self.__desktop_port is None: - self.__desktop_port = self.__findAvailableRandomPort() + _desktop_host = self.__desktop_host if _host_set_by_user else None + logger.debug(f"Desktop port automatically set for host {_desktop_host}") + self.__desktop_port = self.__findAvailableRandomPort(_desktop_host) def __disableDesktop(self): """Procedure to disable exegol desktop feature""" From 20df3f60455abb58b72bbacd49c5fff6a22be26d Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Wed, 27 Sep 2023 23:08:56 +0200 Subject: [PATCH 045/109] fixing automatic port definition statement structure --- exegol/model/ContainerConfig.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 13099dda..7530ee3d 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -530,9 +530,11 @@ def configureDesktop(self, desktop_config: str): logger.critical(f"Your configuration is invalid, please use the following format:[green]mode:host:port[/green]") if self.__desktop_port is None: - _desktop_host = self.__desktop_host if _host_set_by_user else None - logger.debug(f"Desktop port automatically set for host {_desktop_host}") - self.__desktop_port = self.__findAvailableRandomPort(_desktop_host) + logger.debug(f"Desktop port will be set automatically") + if _host_set_by_user: + self.__desktop_port = self.__findAvailableRandomPort(self.__desktop_host) + else: + self.__desktop_port = self.__findAvailableRandomPort() def __disableDesktop(self): """Procedure to disable exegol desktop feature""" From 44090f3cf58c07f0673baed2fb42d1ffbfe6f095 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 28 Sep 2023 18:46:37 +0200 Subject: [PATCH 046/109] Simplify port finder logic --- exegol/model/ContainerConfig.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 7530ee3d..aa5fa346 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -504,7 +504,6 @@ def configureDesktop(self, desktop_config: str): """ self.__desktop_proto = UserConfig().desktop_default_proto self.__desktop_host = "127.0.0.1" if UserConfig().desktop_default_localhost else "0.0.0.0" - _host_set_by_user = False for i, data in enumerate(desktop_config.split(":")): if not data: @@ -519,7 +518,6 @@ def configureDesktop(self, desktop_config: str): elif i == 1 and data: # host logger.debug(f"Desktop host set: {data}") self.__desktop_host = data - _host_set_by_user = True elif i == 2: # port logger.debug(f"Desktop port set: {data}") try: @@ -531,10 +529,7 @@ def configureDesktop(self, desktop_config: str): if self.__desktop_port is None: logger.debug(f"Desktop port will be set automatically") - if _host_set_by_user: - self.__desktop_port = self.__findAvailableRandomPort(self.__desktop_host) - else: - self.__desktop_port = self.__findAvailableRandomPort() + self.__desktop_port = self.__findAvailableRandomPort(self.__desktop_host) def __disableDesktop(self): """Procedure to disable exegol desktop feature""" From c72e683da8aafa085b2b552f1dc700984485a2db Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 28 Sep 2023 19:20:01 +0200 Subject: [PATCH 047/109] Add a check on create mode if the network is available --- exegol/model/ContainerConfig.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index aa5fa346..369ba40b 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -1,7 +1,9 @@ +import errno import logging import os import random import re +import socket import string from datetime import datetime from enum import Enum @@ -12,9 +14,9 @@ from docker.types import Mount from rich.prompt import Prompt +from exegol.config.ConstantConfig import ConstantConfig from exegol.config.EnvInfo import EnvInfo from exegol.config.UserConfig import UserConfig -from exegol.config.ConstantConfig import ConstantConfig from exegol.console.ConsoleFormat import boolFormatter, getColor from exegol.console.ExegolPrompt import Confirm from exegol.console.cli.ParametersManager import ParametersManager @@ -480,7 +482,7 @@ def enableDesktop(self, desktop_config: str = ""): """Procedure to enable exegol desktop feature""" if not self.isDesktopEnabled(): logger.verbose("Config: Enabling exegol desktop") - self.configureDesktop(desktop_config) + self.configureDesktop(desktop_config, create_mode=True) assert self.__desktop_proto is not None assert self.__desktop_host is not None assert self.__desktop_port is not None @@ -498,7 +500,7 @@ def enableDesktop(self, desktop_config: str = ""): # Exposing desktop service self.addPort(port_host=self.__desktop_port, port_container=self.__default_desktop_port[self.__desktop_proto], host_ip=self.__desktop_host) - def configureDesktop(self, desktop_config: str): + def configureDesktop(self, desktop_config: str, create_mode: bool = False): """Configure the exegol desktop feature from user parameters. Accepted format: 'mode:host:port' """ @@ -531,6 +533,19 @@ def configureDesktop(self, desktop_config: str): logger.debug(f"Desktop port will be set automatically") self.__desktop_port = self.__findAvailableRandomPort(self.__desktop_host) + if create_mode: + # Check if the port is available + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + try: + s.bind((self.__desktop_host, self.__desktop_port)) + except socket.error as e: + if e.errno == errno.EADDRINUSE: + logger.critical(f"The port {self.__desktop_host}:{self.__desktop_port} is already in use !") + elif e.errno == errno.EADDRNOTAVAIL: + logger.critical(f"The network {self.__desktop_host}:{self.__desktop_port} is not available !") + else: + logger.critical(f"The supplied network configuration {self.__desktop_host}:{self.__desktop_port} is not available ! ([{e.errno}] {e})") + def __disableDesktop(self): """Procedure to disable exegol desktop feature""" if self.isDesktopEnabled(): @@ -748,13 +763,11 @@ def generateRandomPassword(length: int = 30) -> str: def __findAvailableRandomPort(interface: str = 'localhost') -> int: """Find an available random port. Using the socket system to """ logger.debug(f"Attempting to bind to interface {interface}") - import socket sock = socket.socket() try: sock.bind((interface, 0)) # Using port 0 let the system decide for a random port except OSError as e: - logger.error(f"Unable to bind to interface/port: {e}") - return -1 + logger.critical(f"Unable to bind a port to the interface {interface} ({e})") random_port = sock.getsockname()[1] logger.debug(f"Found available port {random_port}") sock.close() From c9866c2623baf269196935029fbb95b09699ff56 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 28 Sep 2023 19:28:12 +0200 Subject: [PATCH 048/109] Use with format for socket --- exegol/model/ContainerConfig.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 369ba40b..0c7c26c7 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -763,14 +763,13 @@ def generateRandomPassword(length: int = 30) -> str: def __findAvailableRandomPort(interface: str = 'localhost') -> int: """Find an available random port. Using the socket system to """ logger.debug(f"Attempting to bind to interface {interface}") - sock = socket.socket() - try: - sock.bind((interface, 0)) # Using port 0 let the system decide for a random port - except OSError as e: - logger.critical(f"Unable to bind a port to the interface {interface} ({e})") - random_port = sock.getsockname()[1] + with socket.socket() as sock: + try: + sock.bind((interface, 0)) # Using port 0 let the system decide for a random port + except OSError as e: + logger.critical(f"Unable to bind a port to the interface {interface} ({e})") + random_port = sock.getsockname()[1] logger.debug(f"Found available port {random_port}") - sock.close() return random_port # ===== Apply config section ===== From 89d6268edf3b4e1c7b0c84bc499b99fee26d0224 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sat, 30 Sep 2023 22:47:33 +0200 Subject: [PATCH 049/109] better test command --- exegol-resources | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol-resources b/exegol-resources index 4c0dee13..b36c094f 160000 --- a/exegol-resources +++ b/exegol-resources @@ -1 +1 @@ -Subproject commit 4c0dee13ad16fa42947c1677dbd02a1f4456df90 +Subproject commit b36c094f3b38d18a40d707090d2213304f231794 From 42308d6fd89a5032ce7724c4f2f3e151e3f198cb Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Thu, 5 Oct 2023 11:32:00 +0200 Subject: [PATCH 050/109] removing special chars from random root password --- exegol/model/ContainerConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 0c7c26c7..744a31ca 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -756,7 +756,7 @@ def generateRandomPassword(length: int = 30) -> str: """ Generate a new random password. """ - charset = string.ascii_letters + string.digits + string.punctuation.replace("'", "") + charset = string.ascii_letters + string.digits return ''.join(random.choice(charset) for i in range(length)) @staticmethod From 1885fb23ee671ec5810af3aa7acf0fe2fcf824c6 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 11 Oct 2023 23:06:11 +0200 Subject: [PATCH 051/109] opt restriction for Orbstack only Signed-off-by: Dramelac --- exegol/model/ContainerConfig.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 744a31ca..6e9787c0 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -923,7 +923,7 @@ def addVolume(self, if EnvInfo.isMacHost(): # Add support for /etc path_match = str(path) - if path_match.startswith("/opt/"): + if path_match.startswith("/opt/") and EnvInfo.isOrbstack(): msg = f"{EnvInfo.getDockerEngine().value} cannot mount directory from [magenta]/opt/[/magenta] host path." if path_match.endswith("entrypoint.sh") or path_match.endswith("spawn.sh"): msg += " Your exegol installation cannot be stored under this directory." @@ -931,7 +931,7 @@ def addVolume(self, raise CancelOperation(msg) if path_match.startswith("/etc/"): if EnvInfo.isOrbstack(): - raise CancelOperation(f"Orbstack doesn't support sharing /etc files with the container") + raise CancelOperation(f"{EnvInfo.getDockerEngine().value} doesn't support sharing [magenta]/etc[/magenta] files with the container") path_match = path_match.replace("/etc/", "/private/etc/") if EnvInfo.isDockerDesktop(): match = False From 289d753eff19d37d36642a1b29615245dfd87a11 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Wed, 11 Oct 2023 23:08:53 +0200 Subject: [PATCH 052/109] Update exegol env name Signed-off-by: Dramelac --- exegol/model/ContainerConfig.py | 12 ++++++------ exegol/utils/imgsync/spawn.sh | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 6e9787c0..37ae6471 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -49,12 +49,12 @@ class ExegolMetadata(Enum): password = "org.exegol.metadata.passwd" class ExegolEnv(Enum): - user_shell = "START_SHELL" - shell_logging_method = "START_SHELL_LOGGING" - shell_logging_compress = "START_SHELL_COMPRESS" - desktop_protocol = "DESKTOP_PROTO" - desktop_host = "DESKTOP_HOST" - desktop_port = "DESKTOP_PORT" + user_shell = "EXEGOL_START_SHELL" + shell_logging_method = "EXEGOL_START_SHELL_LOGGING" + shell_logging_compress = "EXEGOL_START_SHELL_COMPRESS" + desktop_protocol = "EXEGOL_DESKTOP_PROTO" + desktop_host = "EXEGOL_DESKTOP_HOST" + desktop_port = "EXEGOL_DESKTOP_PORT" # Label features (label name / wrapper method to enable the feature) __label_features = {ExegolFeatures.shell_logging.value: "enableShellLogging", diff --git a/exegol/utils/imgsync/spawn.sh b/exegol/utils/imgsync/spawn.sh index 0802de92..75649de4 100755 --- a/exegol/utils/imgsync/spawn.sh +++ b/exegol/utils/imgsync/spawn.sh @@ -53,11 +53,11 @@ function shell_logging() { } # Find default user shell to use from env var -user_shell=${START_SHELL:-"/bin/zsh"} +user_shell=${EXEGOL_START_SHELL:-"/bin/zsh"} # If shell logging is enable, the method to use is stored in env var -if [ "$START_SHELL_LOGGING" ]; then - shell_logging "$START_SHELL_LOGGING" "$user_shell" "$START_SHELL_COMPRESS" +if [ "$EXEGOL_START_SHELL_LOGGING" ]; then + shell_logging "$EXEGOL_START_SHELL_LOGGING" "$user_shell" "$EXEGOL_START_SHELL_COMPRESS" else $user_shell fi From f80e7e2be435ebc3d920522fe5b88fdefa280a5c Mon Sep 17 00:00:00 2001 From: Dramelac Date: Thu, 12 Oct 2023 14:36:20 +0200 Subject: [PATCH 053/109] Supply exegol container name when hostname is overridden --- exegol/model/ExegolContainer.py | 6 ++++-- exegol/model/ExegolContainerTemplate.py | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index f3b40194..30c1e8c3 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -41,7 +41,8 @@ def __init__(self, docker_container: Container, model: Optional[ExegolContainerT super().__init__(docker_container.name, config=ContainerConfig(docker_container), image=ExegolImage(name=image_name, docker_image=docker_image), - hostname=docker_container.attrs.get('Config', {}).get('Hostname')) + hostname=docker_container.attrs.get('Config', {}).get('Hostname'), + new_container=False) self.image.syncContainerData(docker_container) # At this stage, the container image object has an unknown status because no synchronization with a registry has been done. # This could be done afterwards (with container.image.autoLoad()) if necessary because it takes time. @@ -52,7 +53,8 @@ def __init__(self, docker_container: Container, model: Optional[ExegolContainerT config=ContainerConfig(docker_container), # Rebuild config from docker object to update workspace path image=model.image, - hostname=model.hostname) + hostname=model.hostname, + new_container=False) self.__new_container = True self.image.syncStatus() diff --git a/exegol/model/ExegolContainerTemplate.py b/exegol/model/ExegolContainerTemplate.py index 213f4ae3..7a727015 100644 --- a/exegol/model/ExegolContainerTemplate.py +++ b/exegol/model/ExegolContainerTemplate.py @@ -11,7 +11,7 @@ class ExegolContainerTemplate: """Exegol template class used to create a new container""" - def __init__(self, name: Optional[str], config: ContainerConfig, image: ExegolImage, hostname: Optional[str] = None): + def __init__(self, name: Optional[str], config: ContainerConfig, image: ExegolImage, hostname: Optional[str] = None, new_container: bool = True): if name is None: name = Prompt.ask("[bold blue][?][/bold blue] Enter the name of your new exegol container", default="default") assert name is not None @@ -20,12 +20,14 @@ def __init__(self, name: Optional[str], config: ContainerConfig, image: ExegolIm name = name.lower() self.container_name: str = name if name.startswith("exegol-") else f'exegol-{name}' self.name: str = name.replace('exegol-', '') + self.image: ExegolImage = image + self.config: ContainerConfig = config if hostname: self.hostname: str = hostname + if new_container: + self.config.addEnv("EXEGOL_NAME", self.container_name) else: self.hostname = self.container_name - self.image: ExegolImage = image - self.config: ContainerConfig = config def __str__(self): """Default object text formatter, debug only""" From a4c07b256f856a92abee6c9cad0b23e74c940c5f Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 14 Oct 2023 14:57:48 +0200 Subject: [PATCH 054/109] Using eth0 interface for desktop service when using bridge --- exegol/model/ContainerConfig.py | 5 +++-- exegol/utils/imgsync/entrypoint.sh | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 37ae6471..4da8b67f 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -494,8 +494,9 @@ def enableDesktop(self, desktop_config: str = ""): self.addEnv(self.ExegolEnv.desktop_host.value, self.__desktop_host) self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__desktop_port)) else: - # FIXME : we are exposing the desktop on the network the container is in. Ex: if we're doing VPN, we're opening the desktop through it's network - self.addEnv(self.ExegolEnv.desktop_host.value, "0.0.0.0") + # If we do not specify the host to the container it will automatically choose eth0 interface + # TODO ensure there is an eth0 interface + # Using default port for the service self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__default_desktop_port.get(self.__desktop_proto))) # Exposing desktop service self.addPort(port_host=self.__desktop_port, port_container=self.__default_desktop_port[self.__desktop_proto], host_ip=self.__desktop_host) diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh index 3b390875..042f81bc 100755 --- a/exegol/utils/imgsync/entrypoint.sh +++ b/exegol/utils/imgsync/entrypoint.sh @@ -90,7 +90,7 @@ function run_cmd() { function desktop() { if command -v desktop-start &> /dev/null then - echo "Starting Exegol [green]desktop[/green] with [blue]${DESKTOP_PROTO}[/blue]" + echo "Starting Exegol [green]desktop[/green] with [blue]${EXEGOL_DESKTOP_PROTO}[/blue]" desktop-start &>> ~/.vnc/startup.log # Disable logging sleep 2 # Waiting 2 seconds for the Desktop to start before continuing else From b525d6932c9a1492ee717f1b4d9840ff43ef8e42 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 14 Oct 2023 15:05:57 +0200 Subject: [PATCH 055/109] Using hostname of the container instead --- exegol/utils/DockerUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 4eab322d..cf3562aa 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -110,7 +110,7 @@ def createContainer(cls, model: ExegolContainerTemplate, temporary: bool = False "detach": True, "name": model.container_name, "hostname": model.hostname, - "extra_hosts": {model.hostname: '127.0.0.1'}, + #"extra_hosts": {model.hostname: '127.0.0.1'}, # TODO add this extra_host for network none "devices": model.config.getDevices(), "environment": model.config.getEnvs(), "labels": model.config.getLabels(), From 7c6286d67051c527cb30f9e7b271d5752b85941d Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 14 Oct 2023 15:29:23 +0200 Subject: [PATCH 056/109] Update hostname management and extra_hosts --- exegol/manager/ExegolManager.py | 2 +- exegol/model/ContainerConfig.py | 21 +++++++++++++++++++++ exegol/model/ExegolContainer.py | 6 +++--- exegol/model/ExegolContainerTemplate.py | 8 ++++---- exegol/utils/DockerUtils.py | 4 ++-- 5 files changed, 31 insertions(+), 10 deletions(-) diff --git a/exegol/manager/ExegolManager.py b/exegol/manager/ExegolManager.py index 83e97224..efba442d 100644 --- a/exegol/manager/ExegolManager.py +++ b/exegol/manager/ExegolManager.py @@ -95,7 +95,7 @@ def exec(cls): container.stop(timeout=2) else: # Command is passed at container creation in __createTmpContainer() - logger.success(f"Command executed as entrypoint of the container [green]'{container.hostname}'[/green]") + logger.success(f"Command executed as entrypoint of the container {container.getDisplayName()}") else: container = cast(ExegolContainer, cls.__loadOrCreateContainer(override_container=ParametersManager().selector)) container.exec(command=ParametersManager().exec, as_daemon=ParametersManager().daemon) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 4da8b67f..beb22d1f 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -66,6 +66,7 @@ class ExegolEnv(Enum): def __init__(self, container: Optional[Container] = None): """Container config default value""" + self.hostname = "" self.__enable_gui: bool = False self.__share_timezone: bool = False self.__my_resources: bool = False @@ -81,6 +82,7 @@ def __init__(self, container: Optional[Container] = None): self.__envs: Dict[str, str] = {} self.__labels: Dict[str, str] = {} self.__ports: Dict[str, Optional[Union[int, Tuple[str, int], List[int], List[Dict[str, Union[int, str]]]]]] = {} + self.__extra_host: Dict[str, str] = {} self.interactive: bool = True self.tty: bool = True self.shm_size: str = self.__default_shm_size @@ -848,6 +850,25 @@ def getNetworkMode(self) -> str: """Network mode, docker term getter""" return "host" if self.__network_host else "bridge" + def setExtraHost(self, host: str, ip: str): + """Add or update an extra host to resolv inside the container.""" + self.__extra_host[host] = ip + + def removeExtraHost(self, host: str) -> bool: + """Remove an extra host to resolv inside the container. + Return true if the host was register in the extra_host configuration.""" + return self.__extra_host.pop(host, None) is not None + + def getExtraHost(self): + """Return the extra_host configuration for the container. + Ensure in shared host environment that the container hostname will be correctly resolved to localhost. + Return a dictionary of host and matching IP""" + self.__extra_host = {} + # When using host network mode, you need to add an extra_host to resolve $HOSTNAME + if self.__network_host: + self.setExtraHost(self.hostname, '127.0.0.1') + return self.__extra_host + def getPrivileged(self) -> bool: """Privileged getter""" return self.__privileged diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 30c1e8c3..27197f77 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -53,7 +53,7 @@ def __init__(self, docker_container: Container, model: Optional[ExegolContainerT config=ContainerConfig(docker_container), # Rebuild config from docker object to update workspace path image=model.image, - hostname=model.hostname, + hostname=model.config.hostname, new_container=False) self.__new_container = True self.image.syncStatus() @@ -326,9 +326,9 @@ def __applyXhostACL(self): with console.status(f"Starting XQuartz...", spinner_style="blue"): os.system(f"xhost + localhost > /dev/null") else: - logger.debug(f"Adding xhost ACL to local:{self.hostname}") + logger.debug(f"Adding xhost ACL to local:{self.config.hostname}") # add linux local ACL - os.system(f"xhost +local:{self.hostname} > /dev/null") + os.system(f"xhost +local:{self.config.hostname} > /dev/null") def __updatePasswd(self): """ diff --git a/exegol/model/ExegolContainerTemplate.py b/exegol/model/ExegolContainerTemplate.py index 7a727015..bf601cf5 100644 --- a/exegol/model/ExegolContainerTemplate.py +++ b/exegol/model/ExegolContainerTemplate.py @@ -23,11 +23,11 @@ def __init__(self, name: Optional[str], config: ContainerConfig, image: ExegolIm self.image: ExegolImage = image self.config: ContainerConfig = config if hostname: - self.hostname: str = hostname + self.config.hostname = hostname if new_container: self.config.addEnv("EXEGOL_NAME", self.container_name) else: - self.hostname = self.container_name + self.config.hostname = self.container_name def __str__(self): """Default object text formatter, debug only""" @@ -43,6 +43,6 @@ def rollback(self): def getDisplayName(self) -> str: """Getter of the container's name for TUI purpose""" - if self.container_name != self.hostname: - return f"{self.name} [bright_black]({self.hostname})[/bright_black]" + if self.container_name != self.config.hostname: + return f"{self.name} [bright_black]({self.config.hostname})[/bright_black]" return self.name diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index cf3562aa..fba7e755 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -109,8 +109,8 @@ def createContainer(cls, model: ExegolContainerTemplate, temporary: bool = False "command": command, "detach": True, "name": model.container_name, - "hostname": model.hostname, - #"extra_hosts": {model.hostname: '127.0.0.1'}, # TODO add this extra_host for network none + "hostname": model.config.hostname, + "extra_hosts": model.config.getExtraHost(), "devices": model.config.getDevices(), "environment": model.config.getEnvs(), "labels": model.config.getLabels(), From 22afbdc25a56f1bcba5f0212a09d996333fd1913 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 14 Oct 2023 15:41:13 +0200 Subject: [PATCH 057/109] Add exegol name env to the config enum --- exegol/model/ContainerConfig.py | 1 + exegol/model/ExegolContainerTemplate.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index beb22d1f..3e667199 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -55,6 +55,7 @@ class ExegolEnv(Enum): desktop_protocol = "EXEGOL_DESKTOP_PROTO" desktop_host = "EXEGOL_DESKTOP_HOST" desktop_port = "EXEGOL_DESKTOP_PORT" + exegol_name = "EXEGOL_NAME" # Label features (label name / wrapper method to enable the feature) __label_features = {ExegolFeatures.shell_logging.value: "enableShellLogging", diff --git a/exegol/model/ExegolContainerTemplate.py b/exegol/model/ExegolContainerTemplate.py index bf601cf5..b0f3b678 100644 --- a/exegol/model/ExegolContainerTemplate.py +++ b/exegol/model/ExegolContainerTemplate.py @@ -25,7 +25,7 @@ def __init__(self, name: Optional[str], config: ContainerConfig, image: ExegolIm if hostname: self.config.hostname = hostname if new_container: - self.config.addEnv("EXEGOL_NAME", self.container_name) + self.config.addEnv(ContainerConfig.ExegolEnv.exegol_name.value, self.container_name) else: self.config.hostname = self.container_name From ae6762703dce6f7865b3b0092e39ecb10c63bad4 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 14 Oct 2023 15:55:48 +0200 Subject: [PATCH 058/109] Add exegol randomize service port flag --- exegol/model/ContainerConfig.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 3e667199..aeb8d405 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -49,13 +49,16 @@ class ExegolMetadata(Enum): password = "org.exegol.metadata.passwd" class ExegolEnv(Enum): - user_shell = "EXEGOL_START_SHELL" - shell_logging_method = "EXEGOL_START_SHELL_LOGGING" - shell_logging_compress = "EXEGOL_START_SHELL_COMPRESS" - desktop_protocol = "EXEGOL_DESKTOP_PROTO" - desktop_host = "EXEGOL_DESKTOP_HOST" - desktop_port = "EXEGOL_DESKTOP_PORT" - exegol_name = "EXEGOL_NAME" + # feature + exegol_name = "EXEGOL_NAME" # Supply the name of the container to itself when overriding the hostname + randomize_service_port = "EXEGOL_RANDOMIZE_SERVICE_PORTS" # Enable the randomize port feature when using exegol is network host mode + # config + user_shell = "EXEGOL_START_SHELL" # Set the default shell to use + shell_logging_method = "EXEGOL_START_SHELL_LOGGING" # Enable and select the shell logging method + shell_logging_compress = "EXEGOL_START_SHELL_COMPRESS" # Configure if the logs must be compressed at the end of the shell + desktop_protocol = "EXEGOL_DESKTOP_PROTO" # Configure which desktop module must be started + desktop_host = "EXEGOL_DESKTOP_HOST" # Select the host / ip to expose the desktop service on (container side) + desktop_port = "EXEGOL_DESKTOP_PORT" # Select the port to expose the desktop service on (container side) # Label features (label name / wrapper method to enable the feature) __label_features = {ExegolFeatures.shell_logging.value: "enableShellLogging", @@ -1047,7 +1050,7 @@ def getDevices(self) -> List[str]: return self.__devices def addEnv(self, key: str, value: str): - """Add an environment variable to the container configuration""" + """Add or update an environment variable to the container configuration""" self.__envs[key] = value def removeEnv(self, key: str) -> bool: @@ -1061,6 +1064,9 @@ def removeEnv(self, key: str) -> bool: def getEnvs(self) -> Dict[str, str]: """Envs config getter""" + # When using host network mode, service port must be randomized to avoid conflict between services and container + if self.__network_host: + self.addEnv(self.ExegolEnv.randomize_service_port.value, "true") return self.__envs def getShellEnvs(self) -> List[str]: From 068ea05ba524a6c267b89e6f2283a9b7c5448be1 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 17 Oct 2023 21:53:23 +0200 Subject: [PATCH 059/109] adding estimated remote image size on disk --- exegol/model/ExegolImage.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index 91218da2..3ab0624b 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -51,6 +51,7 @@ def __init__(self, self.__build_date = "[bright_black]N/A[/bright_black]" # Remote image size self.__dl_size: str = "[bright_black]N/A[/bright_black]" + self.__remote_est_size: str = "[bright_black]N/A[/bright_black]" # Local uncompressed image's size self.__disk_size: str = "[bright_black]N/A[/bright_black]" # Remote image ID @@ -73,7 +74,8 @@ def __init__(self, if dockerhub_data: self.__is_remote = True self.__setArch(MetaImages.parseArch(dockerhub_data)) - self.__dl_size = self.__processSize(dockerhub_data.get("size", 0)) + self.__dl_size = self.__processSize(size=dockerhub_data.get("size", 0)) + self.__remote_est_size = self.__processSize(size=dockerhub_data.get("size", 0), compression_factor=2.6) if meta_img and meta_img.meta_id is not None: self.__setDigest(meta_img.meta_id) self.__setLatestRemoteId(meta_img.meta_id) # Meta id is always the latest one @@ -190,7 +192,8 @@ def setMetaImage(self, meta: MetaImages): if fetch_version: meta.version = fetch_version if dockerhub_data is not None: - self.__dl_size = self.__processSize(dockerhub_data.get("size", 0)) + self.__dl_size = self.__processSize(size=dockerhub_data.get("size", 0)) + self.__remote_est_size = self.__processSize(size=dockerhub_data.get("size", 0), compression_factor=2.5) self.__setLatestVersion(meta.version) if meta.meta_id: self.__setLatestRemoteId(meta.meta_id) @@ -455,12 +458,12 @@ def __reorderImages(cls, images: List['ExegolImage']) -> List['ExegolImage']: return result @staticmethod - def __processSize(size: int, precision: int = 1) -> str: + def __processSize(size: int, precision: int = 1, compression_factor: float = 1) -> str: """Text formatter from size number to human-readable size.""" # https://stackoverflow.com/a/32009595 suffixes = ["B", "KB", "MB", "GB", "TB"] suffix_index = 0 - calc: float = size + calc: float = size * compression_factor while calc > 1024 and suffix_index < 4: suffix_index += 1 # increment the index of the suffix calc = calc / 1024 # apply the division @@ -567,7 +570,7 @@ def __setRealSize(self, value: int): def getRealSize(self) -> str: """On-Disk size getter""" - return self.__disk_size + return self.__disk_size if self.__is_install else f"[bright_black]~{self.__remote_est_size}[/bright_black]" def getDownloadSize(self) -> str: """Remote size getter""" @@ -577,7 +580,7 @@ def getDownloadSize(self) -> str: def getSize(self) -> str: """Image size getter. If the image is installed, return the on-disk size, otherwise return the remote size""" - return self.__disk_size if self.__is_install else f"[bright_black]{self.__dl_size} (compressed)[/bright_black]" + return self.__disk_size if self.__is_install else f"[bright_black]~{self.__remote_est_size}[/bright_black]" def getEntrypointConfig(self) -> Optional[Union[str, List[str]]]: """Image's entrypoint configuration getter. From 09ce9e8e88ae00ceefe2bb9145c71d3857a5eb9a Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 17 Oct 2023 22:27:28 +0200 Subject: [PATCH 060/109] adding estimated remote image size on disk when pulling an image --- exegol/model/ExegolImage.py | 4 ++++ exegol/utils/DockerUtils.py | 4 +++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index 3ab0624b..06f0c2dc 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -572,6 +572,10 @@ def getRealSize(self) -> str: """On-Disk size getter""" return self.__disk_size if self.__is_install else f"[bright_black]~{self.__remote_est_size}[/bright_black]" + def getRealSizeRaw(self) -> str: + """On-Disk size getter""" + return self.__disk_size if self.__is_install else self.__remote_est_size + def getDownloadSize(self) -> str: """Remote size getter""" if not self.__is_remote: diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index fba7e755..463e6f91 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -1,4 +1,5 @@ import os +import logging from datetime import datetime from typing import List, Optional, Union, cast @@ -446,7 +447,8 @@ def downloadImage(cls, image: ExegolImage, install_mode: bool = False) -> bool: logger.info(f"{'Installing' if install_mode else 'Updating'} exegol image : {image.getName()}") name = image.updateCheck() if name is not None: - logger.info(f"Starting download. Please wait, this might be (very) long.") + logger.raw(f"[bold blue][*][/bold blue] Pulling compressed image, starting a [cyan1]{image.getDownloadSize()}[/cyan1] download :satellite:{os.linesep}", level=logging.INFO, markup=True, emoji=True) + logger.raw(f"[bold blue][*][/bold blue] Once downloaded and uncompressed, the image will take [cyan1]~{image.getRealSizeRaw()}[/cyan1] on disk :floppy_disk:{os.linesep}", level=logging.INFO, markup=True, emoji=True) logger.debug(f"Downloading {ConstantConfig.IMAGE_NAME}:{name} ({image.getArch()})") try: ExegolTUI.downloadDockerLayer( From 3d937d0bb277bb11807b5cee147fff744101b7d8 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Thu, 19 Oct 2023 10:34:31 +0200 Subject: [PATCH 061/109] changing bug request title prefix --- .github/ISSUE_TEMPLATE/bug_report.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8fb37d07..1812ea5c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,6 +1,6 @@ name: 🐞 Bug report [WRAPPER] description: Report a bug in Exegol WRAPPER to help us improve it -title: "[BUG] " +title: "<title>" labels: - bug body: From 97967b0f4fc131abbbd50341b8a41e56f059af67 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Sat, 21 Oct 2023 18:11:52 +0200 Subject: [PATCH 062/109] Fix git update return code --- exegol/manager/UpdateManager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/exegol/manager/UpdateManager.py b/exegol/manager/UpdateManager.py index aa6df1f2..dba2733c 100644 --- a/exegol/manager/UpdateManager.py +++ b/exegol/manager/UpdateManager.py @@ -172,9 +172,7 @@ def __updateGit(gitUtils: GitUtils) -> bool: if selected_branch is not None and selected_branch != current_branch: gitUtils.checkout(selected_branch) # git pull - gitUtils.update() - logger.empty_line() - return True + return gitUtils.update() @classmethod def checkForWrapperUpdate(cls) -> bool: From 35fe27e441837bb3227a994a3a0d38b249647d3c Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Sat, 21 Oct 2023 18:35:28 +0200 Subject: [PATCH 063/109] Disable VncAuth method --- exegol/model/ExegolContainer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 27197f77..f151713f 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -338,4 +338,3 @@ def __updatePasswd(self): if self.config.getPasswd() is not None: logger.debug(f"Updating the {self.config.getUsername()} password inside the container") self.exec(f"echo '{self.config.getUsername()}:{self.config.getPasswd()}' | chpasswd", quiet=True) - self.exec(f"echo '{self.config.getPasswd()}' | vncpasswd -f > ~/.vnc/passwd", quiet=True) From cebc5f23b587b68348c10786dd6d396fa9665cef Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Sat, 21 Oct 2023 18:58:58 +0200 Subject: [PATCH 064/109] Add username env config --- exegol/model/ContainerConfig.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index aeb8d405..90d091e1 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -54,6 +54,7 @@ class ExegolEnv(Enum): randomize_service_port = "EXEGOL_RANDOMIZE_SERVICE_PORTS" # Enable the randomize port feature when using exegol is network host mode # config user_shell = "EXEGOL_START_SHELL" # Set the default shell to use + exegol_user = "EXEGOL_USERNAME" # Select the username of the container shell_logging_method = "EXEGOL_START_SHELL_LOGGING" # Enable and select the shell logging method shell_logging_compress = "EXEGOL_START_SHELL_COMPRESS" # Configure if the logs must be compressed at the end of the shell desktop_protocol = "EXEGOL_DESKTOP_PROTO" # Configure which desktop module must be started @@ -495,6 +496,7 @@ def enableDesktop(self, desktop_config: str = ""): self.addLabel(self.ExegolFeatures.desktop.value, f"{self.__desktop_proto}:{self.__desktop_host}:{self.__desktop_port}") # Env var are used to send these parameter to the desktop-start script self.addEnv(self.ExegolEnv.desktop_protocol.value, self.__desktop_proto) + self.addEnv(self.ExegolEnv.exegol_user.value, self.getUsername()) if self.__network_host: self.addEnv(self.ExegolEnv.desktop_host.value, self.__desktop_host) @@ -565,6 +567,7 @@ def __disableDesktop(self): self.__desktop_port = None self.removeLabel(self.ExegolFeatures.desktop.value) self.removeEnv(self.ExegolEnv.desktop_protocol.value) + self.removeEnv(self.ExegolEnv.exegol_user.value) self.removeEnv(self.ExegolEnv.desktop_host.value) self.removeEnv(self.ExegolEnv.desktop_port.value) From 969aad118f1041a155e46cbab91e41145374f648 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Sat, 21 Oct 2023 19:05:12 +0200 Subject: [PATCH 065/109] Fix mypy web response --- exegol/utils/WebUtils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/exegol/utils/WebUtils.py b/exegol/utils/WebUtils.py index e43a437c..9f7e81d9 100644 --- a/exegol/utils/WebUtils.py +++ b/exegol/utils/WebUtils.py @@ -127,7 +127,10 @@ def __runRequest(cls, url: str, service_name: str, headers: Optional[Dict] = Non response = requests.request(method=method, url=url, timeout=(5, 10), verify=ParametersManager().verify, headers=headers, data=data) return response except requests.exceptions.HTTPError as e: - logger.error(f"Response error: {e.response.content.decode('utf-8')}") + if e.response.content is not None: + logger.error(f"Response error: {e.response.content.decode('utf-8')}") + else: + logger.error(f"Response error: {e}") except requests.exceptions.ConnectionError as err: logger.debug(f"Error: {err}") error_re = re.search(r"\[Errno [-\d]+]\s?([^']*)('\))+\)*", str(err)) From 7427383a327a2d9214fa377216ed7888b8ae86a1 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Sat, 21 Oct 2023 19:08:48 +0200 Subject: [PATCH 066/109] Fix mypy web response --- exegol/utils/WebUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/utils/WebUtils.py b/exegol/utils/WebUtils.py index 9f7e81d9..47ce893d 100644 --- a/exegol/utils/WebUtils.py +++ b/exegol/utils/WebUtils.py @@ -127,7 +127,7 @@ def __runRequest(cls, url: str, service_name: str, headers: Optional[Dict] = Non response = requests.request(method=method, url=url, timeout=(5, 10), verify=ParametersManager().verify, headers=headers, data=data) return response except requests.exceptions.HTTPError as e: - if e.response.content is not None: + if e.response is not None: logger.error(f"Response error: {e.response.content.decode('utf-8')}") else: logger.error(f"Response error: {e}") From ee6a85ff1c7450ab0e87bb16f2919911f10a7617 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sat, 21 Oct 2023 19:22:03 +0200 Subject: [PATCH 067/109] replacing logger.raw with logger.info for image download intro --- exegol/utils/DockerUtils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 463e6f91..22e67791 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -447,8 +447,8 @@ def downloadImage(cls, image: ExegolImage, install_mode: bool = False) -> bool: logger.info(f"{'Installing' if install_mode else 'Updating'} exegol image : {image.getName()}") name = image.updateCheck() if name is not None: - logger.raw(f"[bold blue][*][/bold blue] Pulling compressed image, starting a [cyan1]{image.getDownloadSize()}[/cyan1] download :satellite:{os.linesep}", level=logging.INFO, markup=True, emoji=True) - logger.raw(f"[bold blue][*][/bold blue] Once downloaded and uncompressed, the image will take [cyan1]~{image.getRealSizeRaw()}[/cyan1] on disk :floppy_disk:{os.linesep}", level=logging.INFO, markup=True, emoji=True) + logger.info(f"Pulling compressed image, starting a [cyan1]~{image.getDownloadSize()}[/cyan1] download :satellite:") + logger.info(f"Once downloaded and uncompressed, the image will take [cyan1]~{image.getRealSizeRaw()}[/cyan1] on disk :floppy_disk:") logger.debug(f"Downloading {ConstantConfig.IMAGE_NAME}:{name} ({image.getArch()})") try: ExegolTUI.downloadDockerLayer( From 3ff0d651d5d14693dbacccc3e1624c5bfc7363c3 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sat, 21 Oct 2023 19:24:48 +0200 Subject: [PATCH 068/109] removing unused logging import --- exegol/utils/DockerUtils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 22e67791..9675671a 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -1,5 +1,4 @@ import os -import logging from datetime import datetime from typing import List, Optional, Union, cast From f53ff45dac0f5263fc7811a51e082f9b186efbf6 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sat, 21 Oct 2023 19:30:42 +0200 Subject: [PATCH 069/109] removing duplicate getSize function --- exegol/console/TUI.py | 4 ++-- exegol/model/ExegolImage.py | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/exegol/console/TUI.py b/exegol/console/TUI.py index 848af017..e1469b82 100644 --- a/exegol/console/TUI.py +++ b/exegol/console/TUI.py @@ -213,9 +213,9 @@ def __buildImageTable(table: Table, data: Sequence[ExegolImage], safe_key: bool image.getRealSize(), image.getBuildDate(), image.getStatus()) else: if safe_key: - table.add_row(str(i + 1), image.getDisplayName(), image.getSize(), image.getStatus()) + table.add_row(str(i + 1), image.getDisplayName(), image.getRealSize(), image.getStatus()) else: - table.add_row(image.getDisplayName(), image.getSize(), image.getStatus()) + table.add_row(image.getDisplayName(), image.getRealSize(), image.getStatus()) @staticmethod def __buildContainerTable(table: Table, data: Sequence[ExegolContainer], safe_key: bool = False): diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index 06f0c2dc..0ea7b0b1 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -582,10 +582,6 @@ def getDownloadSize(self) -> str: return "local" return self.__dl_size - def getSize(self) -> str: - """Image size getter. If the image is installed, return the on-disk size, otherwise return the remote size""" - return self.__disk_size if self.__is_install else f"[bright_black]~{self.__remote_est_size}[/bright_black]" - def getEntrypointConfig(self) -> Optional[Union[str, List[str]]]: """Image's entrypoint configuration getter. Exegol images before 3.x.x don't have any entrypoint set (because /.exegol/entrypoint.sh don't exist yet. In this case, this getter will return None.""" From 600189048925131d4fd80a3f6339e2dea166dc95 Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sun, 22 Oct 2023 18:35:59 +0200 Subject: [PATCH 070/109] Update comments --- exegol/model/ExegolImage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index 0ea7b0b1..c91cc080 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -569,11 +569,11 @@ def __setRealSize(self, value: int): self.__disk_size = self.__processSize(value) def getRealSize(self) -> str: - """On-Disk size getter""" + """Image size getter. If the image is installed, return the on-disk size, otherwise return the remote size""" return self.__disk_size if self.__is_install else f"[bright_black]~{self.__remote_est_size}[/bright_black]" def getRealSizeRaw(self) -> str: - """On-Disk size getter""" + """Image size getter without color. If the image is installed, return the on-disk size, otherwise return the remote size""" return self.__disk_size if self.__is_install else self.__remote_est_size def getDownloadSize(self) -> str: From 26836aace3b22fca0d1efcc1048917c3e6107323 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 24 Oct 2023 10:40:27 +0200 Subject: [PATCH 071/109] initiating custom dockerfiles path for local build --- .../console/cli/actions/ExegolParameters.py | 7 ++++++ exegol/manager/UpdateManager.py | 24 +++++++++++++++---- exegol/utils/DockerUtils.py | 6 ++--- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/exegol/console/cli/actions/ExegolParameters.py b/exegol/console/cli/actions/ExegolParameters.py index 3ccb7712..43b19281 100644 --- a/exegol/console/cli/actions/ExegolParameters.py +++ b/exegol/console/cli/actions/ExegolParameters.py @@ -4,6 +4,7 @@ from exegol.manager.ExegolManager import ExegolManager from exegol.manager.UpdateManager import UpdateManager from exegol.utils.ExeLog import logger +from exegol.config.ConstantConfig import ConstantConfig class Start(Command, ContainerCreation, ContainerSpawnShell): @@ -87,10 +88,16 @@ def __init__(self): metavar="LOGFILE_PATH", action="store", help="Write image building logs to a file.") + self.build_path = Option("--build-path", + dest="build_path", + metavar="DOCKERFILES_PATH", + action="store", + help=f"Path to the dockerfiles and sources.") # Create group parameter for container selection self.groupArgs.append(GroupArg({"arg": self.build_profile, "required": False}, {"arg": self.build_log, "required": False}, + {"arg": self.build_path, "required": False}, title="[bold cyan]Build[/bold cyan] [blue]specific options[/blue]")) self._usages = { diff --git a/exegol/manager/UpdateManager.py b/exegol/manager/UpdateManager.py index dba2733c..1499d5d7 100644 --- a/exegol/manager/UpdateManager.py +++ b/exegol/manager/UpdateManager.py @@ -1,6 +1,7 @@ import re from datetime import datetime, timedelta from typing import Optional, Dict, cast, Tuple, Sequence +from pathlib import Path, PurePath from rich.prompt import Prompt @@ -306,6 +307,7 @@ def __buildSource(cls, build_name: Optional[str] = None) -> str: """build user process : Ask user is he want to update the git source (to get new& updated build profiles), User choice a build name (if not supplied) + User select the path to the dockerfiles User select a build profile Start docker image building Return the name of the built image""" @@ -324,8 +326,20 @@ def __buildSource(cls, build_name: Optional[str] = None) -> str: logger.error("This name is reserved and cannot be used for local build. Please choose another one.") build_name = Prompt.ask("[bold blue][?][/bold blue] Choice a name for your build", default="local") + + # Choose dockerfiles path + build_path: Optional[str] = ParametersManager().build_path + if build_path is None: + if Confirm("Do you want to build from a [blue]custom build path[/blue]?", False): + while True: + build_path = Prompt.ask('Enter the path to the custom Dockerfile(s)') + # TODO: if path is file, only keep the pwd, else check that the dir has dockerfiles in it + else: + build_path = ConstantConfig.build_context_path + logger.debug(f"Using {build_path} as path for dockerfiles") + # Choose dockerfile - profiles = cls.listBuildProfiles() + profiles = cls.listBuildProfiles(profiles_path=build_path) build_profile: Optional[str] = ParametersManager().build_profile build_dockerfile: Optional[str] = None if build_profile is not None: @@ -338,7 +352,7 @@ def __buildSource(cls, build_name: Optional[str] = None) -> str: title="[not italic]:dog: [/not italic][gold3]Profile[/gold3]")) logger.debug(f"Using {build_profile} build profile ({build_dockerfile})") # Docker Build - DockerUtils.buildImage(build_name, build_profile, build_dockerfile) + DockerUtils.buildImage(tag=build_name, build_profile=build_profile, build_dockerfile=build_dockerfile, dockerfile_path=build_path) return build_name @classmethod @@ -348,14 +362,14 @@ def buildAndLoad(cls, tag: str): return DockerUtils.getInstalledImage(build_name) @classmethod - def listBuildProfiles(cls) -> Dict: + def listBuildProfiles(cls, profiles_path: str = ConstantConfig.build_context_path) -> Dict: """List every build profiles available locally Return a dict of options {"key = profile name": "value = dockerfile full name"}""" # Default stable profile profiles = {"full": "Dockerfile"} # List file *.dockerfile is the build context directory - logger.debug(f"Loading build profile from {ConstantConfig.build_context_path}") - docker_files = list(ConstantConfig.build_context_path_obj.glob("*.dockerfile")) + logger.debug(f"Loading build profile from {profiles_path}") + docker_files = list(Path(profiles_path).glob("*.dockerfile")) for file in docker_files: # Convert every file to the dict format filename = file.name diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 9675671a..1a7eb243 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -528,7 +528,7 @@ def removeImage(cls, image: ExegolImage, upgrade_mode: bool = False) -> bool: return False @classmethod - def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerfile: Optional[str] = None): + def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerfile: Optional[str] = None, dockerfile_path: Optional[str] = None): """Build a docker image from source""" if ParametersManager().offline_mode: logger.critical("It's not possible to build a docker image in offline mode. The build process need access to internet ...") @@ -538,7 +538,7 @@ def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerf build_profile = "full" build_dockerfile = "Dockerfile" logger.info("Starting build. Please wait, this will be long.") - logger.verbose(f"Creating build context from [gold]{ConstantConfig.build_context_path}[/gold] with " + logger.verbose(f"Creating build context from [gold]{dockerfile_path}[/gold] with " f"[green][b]{build_profile}[/b][/green] profile ({ParametersManager().arch}).") if EnvInfo.arch != ParametersManager().arch: logger.warning("Building an image for a different host architecture can cause unexpected problems and slowdowns!") @@ -547,7 +547,7 @@ def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerf # tag is the name of the final build # dockerfile is the Dockerfile filename ExegolTUI.buildDockerImage( - cls.__client.api.build(path=ConstantConfig.build_context_path, + cls.__client.api.build(path=dockerfile_path, dockerfile=build_dockerfile, tag=f"{ConstantConfig.IMAGE_NAME}:{tag}", buildargs={"TAG": f"{build_profile}", From 6dd76be381839af55f0d860ee9176f7629d0d941 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Tue, 24 Oct 2023 16:30:37 +0200 Subject: [PATCH 072/109] Update build profile completer + security checks --- exegol/console/cli/ExegolCompleter.py | 16 +++++- .../console/cli/actions/ExegolParameters.py | 1 - exegol/manager/UpdateManager.py | 51 +++++++++++-------- exegol/utils/DockerUtils.py | 2 +- 4 files changed, 46 insertions(+), 24 deletions(-) diff --git a/exegol/console/cli/ExegolCompleter.py b/exegol/console/cli/ExegolCompleter.py index a5e71e3e..e2beca5c 100644 --- a/exegol/console/cli/ExegolCompleter.py +++ b/exegol/console/cli/ExegolCompleter.py @@ -1,6 +1,8 @@ from argparse import Namespace +from pathlib import Path from typing import Tuple +from exegol.config.ConstantConfig import ConstantConfig from exegol.config.DataCache import DataCache from exegol.config.UserConfig import UserConfig from exegol.manager.UpdateManager import UpdateManager @@ -58,7 +60,19 @@ def BuildProfileCompleter(prefix: str, parsed_args: Namespace, **kwargs) -> Tupl # The build profile completer must be trigger only when an image name have been set by user if parsed_args is not None and parsed_args.imagetag is None: return () - data = list(UpdateManager.listBuildProfiles().keys()) + + # Default build path + build_path = ConstantConfig.build_context_path_obj + # Handle custom build path + if parsed_args is not None and parsed_args.build_path is not None: + custom_build_path = Path(parsed_args.build_path).expanduser().absolute() + # Check if we have a directory or a file to select the project directory + if not custom_build_path.is_dir(): + custom_build_path = custom_build_path.parent + build_path = custom_build_path + + # Find profile list + data = list(UpdateManager.listBuildProfiles(profiles_path=build_path).keys()) for obj in data: if prefix and not obj.lower().startswith(prefix.lower()): data.remove(obj) diff --git a/exegol/console/cli/actions/ExegolParameters.py b/exegol/console/cli/actions/ExegolParameters.py index 43b19281..cbe0addb 100644 --- a/exegol/console/cli/actions/ExegolParameters.py +++ b/exegol/console/cli/actions/ExegolParameters.py @@ -78,7 +78,6 @@ def __init__(self): # Create container build arguments self.build_profile = Option("build_profile", metavar="BUILD_PROFILE", - choices=UpdateManager.listBuildProfiles().keys(), nargs="?", action="store", help="Select the build profile used to create a local image.", diff --git a/exegol/manager/UpdateManager.py b/exegol/manager/UpdateManager.py index 1499d5d7..68729fb8 100644 --- a/exegol/manager/UpdateManager.py +++ b/exegol/manager/UpdateManager.py @@ -93,7 +93,7 @@ def updateImage(cls, tag: Optional[str] = None, install_mode: bool = False) -> O def __askToBuild(cls, tag: str) -> Optional[ExegolImage]: """Build confirmation process and image building""" # Need confirmation from the user before starting building. - if ParametersManager().build_profile is not None or \ + if ParametersManager().build_profile is not None or ParametersManager().build_path is not None or \ Confirm("Do you want to build locally a custom image?", default=False): return cls.buildAndLoad(tag) return None @@ -307,18 +307,20 @@ def __buildSource(cls, build_name: Optional[str] = None) -> str: """build user process : Ask user is he want to update the git source (to get new& updated build profiles), User choice a build name (if not supplied) - User select the path to the dockerfiles + User select the path to the dockerfiles (only from CLI parameter) User select a build profile Start docker image building Return the name of the built image""" - # Ask to update git - try: - if ExegolModules().getSourceGit().isAvailable and not ExegolModules().getSourceGit().isUpToDate() and \ - Confirm("Do you want to update image sources (in order to update local build profiles)?", default=True): - cls.updateImageSource() - except AssertionError: - # Catch None git object assertions - logger.warning("Git update is [orange3]not available[/orange3]. Skipping.") + # Don't force update source if using a custom build_path + if ParametersManager().build_path is None: + # Ask to update git + try: + if ExegolModules().getSourceGit().isAvailable and not ExegolModules().getSourceGit().isUpToDate() and \ + Confirm("Do you want to update image sources (in order to update local build profiles)?", default=True): + cls.updateImageSource() + except AssertionError: + # Catch None git object assertions + logger.warning("Git update is [orange3]not available[/orange3]. Skipping.") # Choose tag name blacklisted_build_name = ["stable", "full"] while build_name is None or build_name in blacklisted_build_name: @@ -328,14 +330,19 @@ def __buildSource(cls, build_name: Optional[str] = None) -> str: default="local") # Choose dockerfiles path - build_path: Optional[str] = ParametersManager().build_path - if build_path is None: - if Confirm("Do you want to build from a [blue]custom build path[/blue]?", False): - while True: - build_path = Prompt.ask('Enter the path to the custom Dockerfile(s)') - # TODO: if path is file, only keep the pwd, else check that the dir has dockerfiles in it + # Selecting the default path + build_path = ConstantConfig.build_context_path_obj + if ParametersManager().build_path is not None: + custom_build_path = Path(ParametersManager().build_path).expanduser().absolute() + # Check if we have a directory or a file to select the project directory + if not custom_build_path.is_dir(): + custom_build_path = custom_build_path.parent + # Check if there is Dockerfile profiles + if (custom_build_path / "Dockerfile").is_file() or len(list(custom_build_path.glob("*.dockerfile"))) > 0: + # There is at least one Dockerfile + build_path = custom_build_path else: - build_path = ConstantConfig.build_context_path + logger.critical(f"The directory {custom_build_path.absolute()} doesn't contain any Dockerfile profile.") logger.debug(f"Using {build_path} as path for dockerfiles") # Choose dockerfile @@ -352,7 +359,7 @@ def __buildSource(cls, build_name: Optional[str] = None) -> str: title="[not italic]:dog: [/not italic][gold3]Profile[/gold3]")) logger.debug(f"Using {build_profile} build profile ({build_dockerfile})") # Docker Build - DockerUtils.buildImage(tag=build_name, build_profile=build_profile, build_dockerfile=build_dockerfile, dockerfile_path=build_path) + DockerUtils.buildImage(tag=build_name, build_profile=build_profile, build_dockerfile=build_dockerfile, dockerfile_path=build_path.as_posix()) return build_name @classmethod @@ -362,14 +369,16 @@ def buildAndLoad(cls, tag: str): return DockerUtils.getInstalledImage(build_name) @classmethod - def listBuildProfiles(cls, profiles_path: str = ConstantConfig.build_context_path) -> Dict: + def listBuildProfiles(cls, profiles_path: Path = ConstantConfig.build_context_path_obj) -> Dict: """List every build profiles available locally Return a dict of options {"key = profile name": "value = dockerfile full name"}""" # Default stable profile - profiles = {"full": "Dockerfile"} + profiles = {} + if (profiles_path / "Dockerfile").is_file(): + profiles["full"] = "Dockerfile" # List file *.dockerfile is the build context directory logger.debug(f"Loading build profile from {profiles_path}") - docker_files = list(Path(profiles_path).glob("*.dockerfile")) + docker_files = list(profiles_path.glob("*.dockerfile")) for file in docker_files: # Convert every file to the dict format filename = file.name diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 1a7eb243..b9dfb9a4 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -528,7 +528,7 @@ def removeImage(cls, image: ExegolImage, upgrade_mode: bool = False) -> bool: return False @classmethod - def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerfile: Optional[str] = None, dockerfile_path: Optional[str] = None): + def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerfile: Optional[str] = None, dockerfile_path: str = ConstantConfig.build_context_path): """Build a docker image from source""" if ParametersManager().offline_mode: logger.critical("It's not possible to build a docker image in offline mode. The build process need access to internet ...") From 72266bcd34b0b9af2ba210b3b114ef5d375b0c8f Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:39:11 +0200 Subject: [PATCH 073/109] Fix last update time format --- exegol/manager/UpdateManager.py | 2 +- exegol/model/CacheModels.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/exegol/manager/UpdateManager.py b/exegol/manager/UpdateManager.py index dba2733c..8d43e0cb 100644 --- a/exegol/manager/UpdateManager.py +++ b/exegol/manager/UpdateManager.py @@ -178,7 +178,7 @@ def __updateGit(gitUtils: GitUtils) -> bool: def checkForWrapperUpdate(cls) -> bool: """Check if there is an exegol wrapper update available. Return true if an update is available.""" - logger.debug(f"Last wrapper update check: {DataCache().get_wrapper_data().metadata.get_last_check()}") + logger.debug(f"Last wrapper update check: {DataCache().get_wrapper_data().metadata.get_last_check_text()}") # Skipping update check if DataCache().get_wrapper_data().metadata.is_outdated() and not ParametersManager().offline_mode: logger.debug("Running update check") diff --git a/exegol/model/CacheModels.py b/exegol/model/CacheModels.py index e419313f..1a4ee6e8 100644 --- a/exegol/model/CacheModels.py +++ b/exegol/model/CacheModels.py @@ -20,6 +20,9 @@ def update_last_check(self): def get_last_check(self) -> datetime.datetime: return datetime.datetime.strptime(self.last_check, self.__TIME_FORMAT) + def get_last_check_text(self) -> str: + return self.last_check + def is_outdated(self, days: int = 15, hours: int = 0): """Check if the cache must be considered as expired.""" now = datetime.datetime.now() From 9a54364e1d7e57fbe5e0b9ce71b715bc5e20e12e Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:39:40 +0200 Subject: [PATCH 074/109] Fix exegol-resources submodules for pip env --- exegol/utils/GitUtils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/exegol/utils/GitUtils.py b/exegol/utils/GitUtils.py index 5a3af577..ea60d6c8 100644 --- a/exegol/utils/GitUtils.py +++ b/exegol/utils/GitUtils.py @@ -288,8 +288,6 @@ def __initSubmodules(self): logger.verbose(f"Git {self.getName()} init submodules") # These modules are init / updated manually blacklist_heavy_modules = ["exegol-resources"] - # Submodules dont have depth submodule limits - depth_limit = not self.__is_submodule if self.__gitRepo is None: return with console.status(f"Initialization of git submodules", spinner_style="blue") as s: @@ -299,9 +297,9 @@ def __initSubmodules(self): logger.error(f"Unable to find any git submodule from '{self.getName()}' repository. Check the path in the file {self.__repo_path / '.git'}") return for current_sub in submodules: + logger.debug(f"Loading repo submodules: {current_sub}") # Submodule update are skipped if blacklisted or if the depth limit is set - if current_sub.name in blacklist_heavy_modules or \ - (depth_limit and ('/' in current_sub.name or '\\' in current_sub.name)): + if current_sub.name in blacklist_heavy_modules: continue s.update(status=f"Downloading git submodules [green]{current_sub.name}[/green]") from git.exc import GitCommandError From efe122acb147601671409ade10f3f84980ef7e16 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Tue, 24 Oct 2023 17:46:24 +0200 Subject: [PATCH 075/109] Fix entrypoint script for pip install --- exegol/model/ExegolContainer.py | 8 ++++++-- exegol/utils/imgsync/ImageScriptSync.py | 2 +- setup.py | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index f151713f..609b3a63 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -3,7 +3,7 @@ from datetime import datetime from typing import Optional, Dict, Sequence, Tuple, Union -from docker.errors import NotFound, ImageNotFound +from docker.errors import NotFound, ImageNotFound, APIError from docker.models.containers import Container from exegol.config.EnvInfo import EnvInfo @@ -305,7 +305,11 @@ def postCreateSetup(self, is_temporary: bool = False): self.__container.put_archive("/", ImageScriptSync.getImageSyncTarData(include_entrypoint=True)) if self.__container.status.lower() == "created": self.__start_container() - self.__updatePasswd() + try: + self.__updatePasswd() + except APIError as e: + if "is not running" in e.explanation: + logger.critical("An unexpected error occurred. Exegol cannot start the container after its creation...") def __applyXhostACL(self): """ diff --git a/exegol/utils/imgsync/ImageScriptSync.py b/exegol/utils/imgsync/ImageScriptSync.py index 36540609..646f0762 100644 --- a/exegol/utils/imgsync/ImageScriptSync.py +++ b/exegol/utils/imgsync/ImageScriptSync.py @@ -30,7 +30,7 @@ def getImageSyncTarData(include_entrypoint: bool = False, include_spawn: bool = entrypoint_script_path = ConstantConfig.entrypoint_context_path_obj logger.debug(f"Entrypoint script path: {str(entrypoint_script_path)}") if not entrypoint_script_path.is_file(): - logger.error("Unable to find the entrypoint script! Your Exegol installation is probably broken...") + logger.critical("Unable to find the entrypoint script! Your Exegol installation is probably broken...") return None with open(entrypoint_script_path, 'rb') as f: raw = f.read() diff --git a/setup.py b/setup.py index eedac5ff..4e48b95e 100644 --- a/setup.py +++ b/setup.py @@ -24,8 +24,8 @@ data_files_dict[key] = [] data_files_dict[key].append(str(path)) ## exegol scripts pushed from the wrapper -data_files_dict["exegol-imgsync"] = ["exegol/utils/imgsync/entrypoint.sh"] -data_files_dict["exegol-imgsync"] = ["exegol/utils/imgsync/spawn.sh"] +data_files_dict["exegol-imgsync"] = ["exegol/utils/imgsync/entrypoint.sh", + "exegol/utils/imgsync/spawn.sh"] # Dict to tuple for k, v in data_files_dict.items(): From 200658119728031a4b8da64885d77f37626e2ae8 Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sat, 28 Oct 2023 19:07:11 +0200 Subject: [PATCH 076/109] Add comment for alpha & beta re-upload Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/model/ExegolContainer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index 609b3a63..d62c66c5 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -288,7 +288,9 @@ def __check_start_version(self): if not self.config.isWrapperStartShared(): # If the spawn.sh if not shared, the version must be compared and the script updated current_start = ImageScriptSync.getCurrentStartVersion() - container_version = self.__container.exec_run(["/bin/bash", "-c", "egrep '^# Spawn Version:[0-9]+$' /.exegol/spawn.sh 2&>/dev/null || echo ':0' | cut -d ':' -f2"]).output.decode("utf-8").strip() + # Try to parse the spawn version of the container. If an alpha or beta version is in use, the script will always be updated. + spawn_parsing_cmd = ["/bin/bash", "-c", "egrep '^# Spawn Version:[0-9]+$' /.exegol/spawn.sh 2&>/dev/null || echo ':0' | cut -d ':' -f2"] + container_version = self.__container.exec_run(spawn_parsing_cmd).output.decode("utf-8").strip() if current_start != container_version: logger.debug(f"Updating spawn.sh script from version {container_version} to version {current_start}") self.__container.put_archive("/", ImageScriptSync.getImageSyncTarData(include_spawn=True)) From ab11cf526aac3eb712f9f674ce5d2ddbbaeec224 Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sat, 28 Oct 2023 20:01:41 +0200 Subject: [PATCH 077/109] Disable startup sequence for legacy container Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/model/ContainerConfig.py | 3 ++- exegol/model/ExegolContainer.py | 37 +++++++++++++++++---------------- exegol/utils/GuiUtils.py | 7 ++++--- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 90d091e1..044cf1f1 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -31,7 +31,6 @@ class ContainerConfig: """Configuration class of an exegol container""" # Default hardcoded value - __default_entrypoint_legacy = "bash" __default_entrypoint = ["/bin/bash", "/.exegol/entrypoint.sh"] __default_shm_size = "64M" @@ -98,6 +97,7 @@ def __init__(self, container: Optional[Container] = None): self.__vpn_path: Optional[Union[Path, PurePath]] = None self.__shell_logging: bool = False # Entrypoint features + self.legacy_entrypoint: bool = True self.__vpn_parameters: Optional[str] = None self.__run_cmd: bool = False self.__endless_container: bool = True @@ -128,6 +128,7 @@ def __parseContainerConfig(self, container: Container): self.__parseEnvs(container_config.get("Env", [])) self.__parseLabels(container_config.get("Labels", {})) self.interactive = container_config.get("OpenStdin", True) + self.legacy_entrypoint = container_config.get("Entrypoint") is None self.__enable_gui = False for env in self.__envs: if "DISPLAY" in env: diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index d62c66c5..aedcaef4 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -117,24 +117,25 @@ def __start_container(self): with console.status(f"Waiting to start {self.name}", spinner_style="blue") as progress: start_date = datetime.utcnow() self.__container.start() - try: - # Try to find log / startup messages. Will time out after 2 seconds if the image don't support status update through container logs. - for line in ContainerLogStream(self.__container, start_date=start_date, timeout=2): - # Once the last log "READY" is received, the startup sequence is over and the execution can continue - if line == "READY": - break - elif line.startswith('[W]'): - line = line.replace('[W]', '') - logger.warning(line) - elif line.startswith('[E]'): - line = line.replace('[E]', '') - logger.error(line) - else: - logger.verbose(line) - progress.update(status=f"[blue]\[Startup][/blue] {line}") - except KeyboardInterrupt: - # User can cancel startup logging with ctrl+C - logger.warning("User skip startup status updates. Spawning a shell now.") + if not self.config.legacy_entrypoint: # TODO improve startup compatibility check + try: + # Try to find log / startup messages. Will time out after 2 seconds if the image don't support status update through container logs. + for line in ContainerLogStream(self.__container, start_date=start_date, timeout=2): + # Once the last log "READY" is received, the startup sequence is over and the execution can continue + if line == "READY": + break + elif line.startswith('[W]'): + line = line.replace('[W]', '') + logger.warning(line) + elif line.startswith('[E]'): + line = line.replace('[E]', '') + logger.error(line) + else: + logger.verbose(line) + progress.update(status=f"[blue]\[Startup][/blue] {line}") + except KeyboardInterrupt: + # User can cancel startup logging with ctrl+C + logger.warning("User skip startup status updates. Spawning a shell now.") def stop(self, timeout: int = 10): """Stop the docker container""" diff --git a/exegol/utils/GuiUtils.py b/exegol/utils/GuiUtils.py index 3bf6be83..7a7eb424 100644 --- a/exegol/utils/GuiUtils.py +++ b/exegol/utils/GuiUtils.py @@ -7,9 +7,9 @@ from pathlib import Path from typing import Optional +from exegol.config.EnvInfo import EnvInfo from exegol.console.ExegolPrompt import Confirm from exegol.exceptions.ExegolExceptions import CancelOperation -from exegol.config.EnvInfo import EnvInfo from exegol.utils.ExeLog import logger, console @@ -70,8 +70,9 @@ def getDisplayEnv(cls) -> str: # Add ENV check is case of user don't have it, which will mess up GUI (X11 sharing) if fallback does not work # @see https://github.com/ThePorgs/Exegol/issues/148 - if os.getenv("DISPLAY") is None: - logger.warning("The DISPLAY environment variable is not set on your host. This can prevent GUI apps to start through X11 sharing") + if not EnvInfo.is_windows_shell: + if os.getenv("DISPLAY") is None: + logger.warning("The DISPLAY environment variable is not set on your host. This can prevent GUI apps to start through X11 sharing") # DISPLAY var is fetch from the current user environment. If it doesn't exist, using ':0'. return os.getenv('DISPLAY', ":0") From 2a8c05ca6ecc99e0e9d241b88484f1b150f6273a Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sat, 28 Oct 2023 20:03:14 +0200 Subject: [PATCH 078/109] Mute warning when nothing to change Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/utils/imgsync/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh index 042f81bc..74d3daa9 100755 --- a/exegol/utils/imgsync/entrypoint.sh +++ b/exegol/utils/imgsync/entrypoint.sh @@ -3,7 +3,7 @@ trap shutdown SIGTERM function exegol_init() { - usermod -s "/.exegol/spawn.sh" root + usermod -s "/.exegol/spawn.sh" root > /dev/null } # Function specific From 366afa7648c66377e2f35c7a8cadb93077bbcaeb Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sun, 29 Oct 2023 14:02:40 +0100 Subject: [PATCH 079/109] Fix rollback of non-empty workspace Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/model/ContainerConfig.py | 6 ++++-- exegol/utils/DockerUtils.py | 11 ++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 044cf1f1..4c4c09e9 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -728,10 +728,12 @@ def rollback_preparation(self, share_name: str): """Undo preparation in case of container creation failure""" if self.__workspace_custom_path is None and not self.__disable_workspace: # Remove dedicated workspace volume - logger.info("Rollback: removing dedicated workspace directory") directory_path = UserConfig().private_volume_path.joinpath(share_name) - if directory_path.is_dir(): + if directory_path.is_dir() and len(list(directory_path.iterdir())) == 0: + logger.info("Rollback: removing dedicated workspace directory") directory_path.rmdir() + else: + logger.warning("Rollback: the workspace directory isn't empty, it will NOT be removed automatically") def entrypointRunCmd(self, endless_mode=False): """Enable the run_cmd feature of the entrypoint. This feature execute the command stored in the $CMD container environment variables. diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index b9dfb9a4..0e984bfa 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -9,7 +9,10 @@ from docker.models.volumes import Volume from requests import ReadTimeout +from exegol.config.ConstantConfig import ConstantConfig from exegol.config.DataCache import DataCache +from exegol.config.EnvInfo import EnvInfo +from exegol.config.UserConfig import UserConfig from exegol.console.TUI import ExegolTUI from exegol.console.cli.ParametersManager import ParametersManager from exegol.exceptions.ExegolExceptions import ObjectNotFound @@ -17,10 +20,7 @@ from exegol.model.ExegolContainerTemplate import ExegolContainerTemplate from exegol.model.ExegolImage import ExegolImage from exegol.model.MetaImages import MetaImages -from exegol.config.ConstantConfig import ConstantConfig -from exegol.config.EnvInfo import EnvInfo from exegol.utils.ExeLog import logger, console, ExeLog -from exegol.config.UserConfig import UserConfig from exegol.utils.WebUtils import WebUtils @@ -133,8 +133,9 @@ def createContainer(cls, model: ExegolContainerTemplate, temporary: bool = False container = docker_create_function(**docker_args) except APIError as err: message = err.explanation.decode('utf-8').replace('[', '\\[') if type(err.explanation) is bytes else err.explanation - message = message.replace('[', '\\[') - logger.error(message) + if message is not None: + message = message.replace('[', '\\[') + logger.error(message) logger.debug(err) model.rollback() try: From de9e7e1f709edf0de7b095a39a75f80b080de63d Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sun, 29 Oct 2023 14:13:47 +0100 Subject: [PATCH 080/109] Fix Path parsing on Windows Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/utils/FsUtils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/exegol/utils/FsUtils.py b/exegol/utils/FsUtils.py index 73ddcb58..e8f998a1 100644 --- a/exegol/utils/FsUtils.py +++ b/exegol/utils/FsUtils.py @@ -2,7 +2,7 @@ import re import stat import subprocess -from pathlib import Path, PurePosixPath, PurePath +from pathlib import Path, PurePath from typing import Optional from exegol.config.EnvInfo import EnvInfo @@ -20,7 +20,7 @@ def parseDockerVolumePath(source: str) -> PurePath: return src_path else: # Remove docker mount path if exist - return PurePosixPath(source.replace('/run/desktop/mnt/host', '')) + return PurePath(source.replace('/run/desktop/mnt/host', '')) def resolvPath(path: Path) -> str: From cef1345497c3773146cc6ae711f2df9665266de0 Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sun, 29 Oct 2023 14:36:57 +0100 Subject: [PATCH 081/109] Fix WSLg path test Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/utils/GuiUtils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/exegol/utils/GuiUtils.py b/exegol/utils/GuiUtils.py index 7a7eb424..356f1d5c 100644 --- a/exegol/utils/GuiUtils.py +++ b/exegol/utils/GuiUtils.py @@ -263,10 +263,12 @@ def __wslg_installed(cls) -> bool: :return: bool """ if EnvInfo.current_platform == "WSL": - if Path("/mnt/host/wslg/versions.txt").is_file(): + if (Path("/mnt/host/wslg/versions.txt").is_file() or + Path("/mnt/wslg/versions.txt").is_file()): return True else: - if cls.__wsl_test("/mnt/host/wslg/versions.txt", name=cls.__distro_name): + if (cls.__wsl_test("/mnt/host/wslg/versions.txt", name=cls.__distro_name) or + cls.__wsl_test("/mnt/wslg/versions.txt", name=cls.__distro_name)): return True logger.debug("WSLg check failed.. Trying a fallback check method.") return cls.__wsl_test("/mnt/host/wslg/versions.txt") or cls.__wsl_test("/mnt/wslg/versions.txt", name=None) From c612d781dbf49f046412196e7aca56f15e4f8e37 Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sun, 29 Oct 2023 14:58:12 +0100 Subject: [PATCH 082/109] Add support for WSLg on Win10 Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/config/EnvInfo.py | 21 +-------------------- exegol/utils/GuiUtils.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 30 deletions(-) diff --git a/exegol/config/EnvInfo.py b/exegol/config/EnvInfo.py index ff54b9fa..f7f46da3 100644 --- a/exegol/config/EnvInfo.py +++ b/exegol/config/EnvInfo.py @@ -1,8 +1,5 @@ import json import platform -import re -import shutil -import subprocess from enum import Enum from typing import Optional, Any, List @@ -114,24 +111,8 @@ def getWindowsRelease(cls) -> str: if cls.is_windows_shell: # From a Windows shell, python supply an approximate (close enough) version of windows cls.__windows_release = platform.win32_ver()[1] - elif cls.current_platform == "WSL": - # From a WSL shell, we must create a process to retrieve the host's version - # Find version using MS-DOS command 'ver' - if not shutil.which("cmd.exe"): - logger.critical("cmd.exe is not accessible from your WSL environment!") - proc = subprocess.Popen(["cmd.exe", "/c", "ver"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) - proc.wait() - assert proc.stdout is not None - # Try to match Windows version - matches = re.search(r"version (\d+\.\d+\.\d+)(\.\d*)?", proc.stdout.read().decode('utf-8')) - if matches: - # Select match 1 and apply to the attribute - cls.__windows_release = matches.group(1) - else: - # If there is any match, fallback to empty - cls.__windows_release = "" else: - cls.__windows_release = "" + cls.__windows_release = "Unknown" return cls.__windows_release @classmethod diff --git a/exegol/utils/GuiUtils.py b/exegol/utils/GuiUtils.py index 356f1d5c..a5f89cf5 100644 --- a/exegol/utils/GuiUtils.py +++ b/exegol/utils/GuiUtils.py @@ -200,12 +200,12 @@ def __windowsGuiChecks(cls) -> bool: return True elif cls.__wslg_eligible(): logger.info("[green]WSLg[/green] is available on your system but [orange3]not installed[/orange3].") - logger.info("Make sure, [green]WSLg[/green] is installed on your Windows by running " - "'wsl --update' as [orange3]admin[/orange3].") - return True + logger.info("Make sure, your Windows is [green]up-to-date[/green] and [green]WSLg[/green] is installed on " + "your host by running 'wsl --update' as [orange3]admin[/orange3].") + return False logger.debug("WSLg is [orange3]not available[/orange3]") logger.warning("Display sharing is [orange3]not supported[/orange3] on your version of Windows. " - "You need to upgrade to [turquoise2]Windows 11[/turquoise2].") + "You need to upgrade to [turquoise2]Windows 10+[/turquoise2].") return False @staticmethod @@ -279,19 +279,18 @@ def __wslg_eligible() -> bool: Check if the current Windows version support WSLg :return: """ + if EnvInfo.current_platform == "WSL": + # WSL is only available on Windows 10 & 11 so WSLg can be installed. + return True try: os_version_raw, _, build_number_raw = EnvInfo.getWindowsRelease().split('.')[:3] except ValueError: logger.debug(f"Impossible to find the version of windows: '{EnvInfo.getWindowsRelease()}'") logger.error("Exegol can't know if your [orange3]version of Windows[/orange3] can support dockerized GUIs (X11 sharing).") return False - # Available from Windows 10 Build 21364 - # Available from Windows 11 Build 22000 + # Available for Windows 10 & 11 os_version = int(os_version_raw) - build_number = int(build_number_raw) - if os_version == 10 and build_number >= 21364: - return True - elif os_version > 10: + if os_version >= 10: return True return False From 74887246f8b465ba1b199e4403e3fa2eebdc7575 Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sun, 29 Oct 2023 15:09:37 +0100 Subject: [PATCH 083/109] Add todo for windows cross env support Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/model/ContainerConfig.py | 1 + 1 file changed, 1 insertion(+) diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 4c4c09e9..79fd6502 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -226,6 +226,7 @@ def __parseMounts(self, mounts: Optional[List[Dict]], name: str): obj_path = cast(PurePath, src_path) logger.debug(f"└── Loading workspace volume source : {obj_path}") self.__disable_workspace = False + # TODO use label to identify manage workspace and support cross env removing if obj_path is not None and obj_path.name == name and \ (obj_path.parent.name == "shared-data-volumes" or obj_path.parent == UserConfig().private_volume_path): # Check legacy path and new custom path logger.debug("└── Private workspace detected") From 7e6cd0c4b32bba098b0305cd78e24727614459b0 Mon Sep 17 00:00:00 2001 From: Dramelac <Dramelac@users.noreply.github.com> Date: Sun, 29 Oct 2023 15:45:55 +0100 Subject: [PATCH 084/109] Update windows path Signed-off-by: Dramelac <Dramelac@users.noreply.github.com> --- exegol/utils/GuiUtils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/exegol/utils/GuiUtils.py b/exegol/utils/GuiUtils.py index a5f89cf5..62ee1f95 100644 --- a/exegol/utils/GuiUtils.py +++ b/exegol/utils/GuiUtils.py @@ -299,7 +299,7 @@ def __find_wsl_distro(cls) -> str: distro_name = "" # these distros cannot be used to load WSLg socket blacklisted_distro = ["docker-desktop", "docker-desktop-data"] - ret = subprocess.Popen(["C:\Windows\system32\wsl.exe", "-l"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + ret = subprocess.Popen(["C:\\Windows\\system32\\wsl.exe", "-l"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) # Wait for WSL process to end ret.wait() if ret.returncode == 0: @@ -353,7 +353,7 @@ def __find_wsl_distro(cls) -> str: @classmethod def __create_default_wsl(cls) -> bool: logger.info("Creating Ubuntu WSL distribution. Please wait.") - ret = subprocess.Popen(["C:\Windows\system32\wsl.exe", "--install", "-d", "Ubuntu"], stderr=subprocess.PIPE) + ret = subprocess.Popen(["C:\\Windows\\system32\\wsl.exe", "--install", "-d", "Ubuntu"], stderr=subprocess.PIPE) ret.wait() logger.info("Please follow installation instructions on the new window.") if ret.returncode != 0: @@ -369,7 +369,7 @@ def __create_default_wsl(cls) -> bool: if docker_settings is not None and docker_settings.get("enableIntegrationWithDefaultWslDistro", False): logger.verbose("Set WSL Ubuntu as default to automatically enable docker integration") # Set new WSL distribution as default to start it and enable docker integration - ret = subprocess.Popen(["C:\Windows\system32\wsl.exe", "-s", "Ubuntu"], stderr=subprocess.PIPE) + ret = subprocess.Popen(["C:\\Windows\\system32\\wsl.exe", "-s", "Ubuntu"], stderr=subprocess.PIPE) ret.wait() # Wait for the docker integration (10 try, 1 sec apart) with console.status("Waiting for the activation of the docker integration", spinner_style="blue"): From 0e678132cd2f8ba90285380098d73ee1eee4f88e Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Tue, 14 Nov 2023 13:15:50 +0100 Subject: [PATCH 085/109] Suppress external lib from traceback --- exegol/manager/ExegolController.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/exegol/manager/ExegolController.py b/exegol/manager/ExegolController.py index 0a1a23b2..169594b1 100644 --- a/exegol/manager/ExegolController.py +++ b/exegol/manager/ExegolController.py @@ -1,9 +1,11 @@ try: - from git.exc import GitCommandError + import docker + import requests + import git + from exegol.utils.ExeLog import logger, ExeLog, console from exegol.console.cli.ParametersManager import ParametersManager from exegol.console.cli.actions.ExegolParameters import Command - from exegol.utils.ExeLog import logger, ExeLog, console except ModuleNotFoundError as e: print("Mandatory dependencies are missing:", e) print("Please install them with python3 -m pip install --upgrade -r requirements.txt") @@ -60,11 +62,11 @@ def main(): except KeyboardInterrupt: logger.empty_line() logger.info("Exiting") - except GitCommandError as e: + except git.exc.GitCommandError as git_error: print_exception_banner() - error = e.stderr.strip().split(": ")[-1].strip("'") - logger.critical(f"A critical error occurred while running this git command: {' '.join(e.command)} => {error}") + error = git_error.stderr.strip().split(": ")[-1].strip("'") + logger.critical(f"A critical error occurred while running this git command: {' '.join(git_error.command)} => {error}") except Exception: print_exception_banner() - console.print_exception(show_locals=True) + console.print_exception(show_locals=True, suppress=[docker, requests, git]) exit(1) From f253741129e837a9fcddc5611fa3c10130db96be Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:27:56 +0100 Subject: [PATCH 086/109] Fix root bypass permission error --- exegol/utils/GitUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/utils/GitUtils.py b/exegol/utils/GitUtils.py index ea60d6c8..868d7b56 100644 --- a/exegol/utils/GitUtils.py +++ b/exegol/utils/GitUtils.py @@ -44,7 +44,7 @@ def __init__(self, elif sys.platform == "win32": # Skip next platform specific code (temp fix for mypy static code analysis) pass - elif not EnvInfo.is_windows_shell and test_git_dir.lstat().st_uid != os.getuid(): + elif not EnvInfo.is_windows_shell and os.getuid() != 0 and test_git_dir.lstat().st_uid != os.getuid(): raise PermissionError(test_git_dir.owner()) except ReferenceError: if self.__git_name == "wrapper": From ad6f2622b1410d538c59d2c6009dec0983b12635 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Sun, 26 Nov 2023 19:07:32 +0100 Subject: [PATCH 087/109] Add static ip to handle GUI with VPN new default route --- exegol/utils/imgsync/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh index 74d3daa9..06bc482c 100755 --- a/exegol/utils/imgsync/entrypoint.sh +++ b/exegol/utils/imgsync/entrypoint.sh @@ -62,10 +62,12 @@ function shutdown() { function _resolv_docker_host() { # On docker desktop host, resolving the host.docker.internal before starting a VPN connection for GUI applications - docker_ip=$(getent hosts host.docker.internal | head -n1 | awk '{ print $1 }') + docker_ip=$(getent ahostsv4 host.docker.internal | head -n1 | awk '{ print $1 }') if [ "$docker_ip" ]; then # Add docker internal host resolution to the hosts file to preserve access to the X server echo "$docker_ip host.docker.internal" >>/etc/hosts + # If the container share the host networks, no need to add a static mapping + ip route list match "$docker_ip" table all | grep -v default || ip route add "$docker_ip/32" $(ip route list | grep default | head -n1 | grep -Eo '(via [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ )?dev [a-zA-Z0-9]+') || echo '[W]Exegol cannot add a static route to resolv your host X11 server. GUI applications may not work.' fi } From 1472d359177bd28e710c8a7dd18b40bd427b0eb5 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Mon, 27 Nov 2023 17:59:07 +0100 Subject: [PATCH 088/109] Fix symlink mismatch from host --- exegol/utils/FsUtils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/exegol/utils/FsUtils.py b/exegol/utils/FsUtils.py index e8f998a1..5c70acff 100644 --- a/exegol/utils/FsUtils.py +++ b/exegol/utils/FsUtils.py @@ -68,7 +68,12 @@ def setGidPermission(root_folder: Path): perm_alert = True for sub_item in root_folder.rglob('*'): # Find every subdirectory - if not sub_item.is_dir(): + try: + if not sub_item.is_dir(): + continue + except PermissionError: + if not sub_item.is_symlink(): + logger.error(f"Permission denied when trying to resolv {str(sub_item)}") continue # If the permission is already set, skip if sub_item.stat().st_mode & stat.S_ISGID: From 84b05a84a518496e126015f42aa5008310a5af01 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Tue, 12 Dec 2023 00:57:05 +0100 Subject: [PATCH 089/109] adding beta mention for the desktop --- exegol/console/cli/actions/GenericParameters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/console/cli/actions/GenericParameters.py b/exegol/console/cli/actions/GenericParameters.py index 30dedca0..ca584b30 100644 --- a/exegol/console/cli/actions/GenericParameters.py +++ b/exegol/console/cli/actions/GenericParameters.py @@ -279,4 +279,4 @@ def __init__(self, groupArgs: List[GroupArg]): completer=DesktopConfigCompleter) groupArgs.append(GroupArg({"arg": self.desktop, "required": False}, {"arg": self.desktop_config, "required": False}, - title="[blue]Container creation Desktop options[/blue]")) + title="[blue]Container creation Desktop options[/blue] [spring_green1](beta)[/spring_green1]")) From aaf8057052ffe3d42543c288029d44cb9c9a70f4 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Wed, 13 Dec 2023 23:29:50 +0100 Subject: [PATCH 090/109] Detect using docker desktop on linux --- exegol/config/EnvInfo.py | 9 ++++++--- exegol/utils/DockerUtils.py | 8 ++++---- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/exegol/config/EnvInfo.py b/exegol/config/EnvInfo.py index f7f46da3..7c3de370 100644 --- a/exegol/config/EnvInfo.py +++ b/exegol/config/EnvInfo.py @@ -21,7 +21,7 @@ class DockerEngine(Enum): """Dictionary class for static Docker engine name""" WLS2 = "WSL2" HYPERV = "Hyper-V" - MAC = "Docker desktop" + DOCKER_DESKTOP = "Docker desktop" ORBSTACK = "Orbstack" LINUX = "Kernel" @@ -85,8 +85,8 @@ def initData(cls, docker_info): cls.__docker_host_os = cls.HostOs.WINDOWS elif cls.__is_docker_desktop: # If docker desktop is detected but not a Windows engine/kernel, it's (probably) a mac - cls.__docker_engine = cls.DockerEngine.MAC - cls.__docker_host_os = cls.HostOs.MAC + cls.__docker_engine = cls.DockerEngine.DOCKER_DESKTOP + cls.__docker_host_os = cls.HostOs.LINUX if "linuxkit" in docker_kernel else cls.HostOs.MAC elif is_orbstack: # Orbstack is only available on Mac cls.__docker_engine = cls.DockerEngine.ORBSTACK @@ -96,6 +96,9 @@ def initData(cls, docker_info): cls.__docker_engine = cls.DockerEngine.LINUX cls.__docker_host_os = cls.HostOs.LINUX + if cls.__docker_engine == cls.DockerEngine.DOCKER_DESKTOP and cls.__docker_host_os == cls.HostOs.LINUX: + logger.warning(f"Using Docker Desktop on Linux is not officially supported !") + @classmethod def getHostOs(cls) -> HostOs: """Return Host OS diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 0e984bfa..a032bf89 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -41,11 +41,11 @@ class DockerUtils: EnvInfo.initData(__daemon_info) except DockerException as err: if 'ConnectionRefusedError' in str(err): - logger.critical("Unable to connect to docker (from env config). Is docker running on your machine? " - "Exiting.") + logger.critical(f"Unable to connect to docker (from env config). Is docker running on your machine? Exiting.{os.linesep}" + f" Check documentation for help: https://exegol.readthedocs.io/en/latest/getting-started/faq.html#unable-to-connect-to-docker") elif 'FileNotFoundError' in str(err): - logger.critical("Unable to connect to docker. Is docker installed on your machine? " - "Exiting.") + logger.critical(f"Unable to connect to docker. Is docker installed on your machine? Exiting.{os.linesep}" + f" Check documentation for help: https://exegol.readthedocs.io/en/latest/getting-started/faq.html#unable-to-connect-to-docker") else: logger.error(err) logger.critical( From b0de254898e1ae0ed675dea1b41d2aa3d1b9de74 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Thu, 14 Dec 2023 19:26:01 +0100 Subject: [PATCH 091/109] Fix mac OS detection --- exegol/config/EnvInfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/config/EnvInfo.py b/exegol/config/EnvInfo.py index 7c3de370..d0eb1a17 100644 --- a/exegol/config/EnvInfo.py +++ b/exegol/config/EnvInfo.py @@ -86,7 +86,7 @@ def initData(cls, docker_info): elif cls.__is_docker_desktop: # If docker desktop is detected but not a Windows engine/kernel, it's (probably) a mac cls.__docker_engine = cls.DockerEngine.DOCKER_DESKTOP - cls.__docker_host_os = cls.HostOs.LINUX if "linuxkit" in docker_kernel else cls.HostOs.MAC + cls.__docker_host_os = cls.HostOs.MAC if cls.is_mac_shell else cls.HostOs.LINUX elif is_orbstack: # Orbstack is only available on Mac cls.__docker_engine = cls.DockerEngine.ORBSTACK From 7ba69a1f5de536eb5ce44b4e8a6579287514ecd2 Mon Sep 17 00:00:00 2001 From: Shutdown <40902872+ShutdownRepo@users.noreply.github.com> Date: Sun, 17 Dec 2023 00:46:42 +0100 Subject: [PATCH 092/109] changing image install message --- exegol/utils/DockerUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index a032bf89..f684297a 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -457,7 +457,7 @@ def downloadImage(cls, image: ExegolImage, install_mode: bool = False) -> bool: stream=True, decode=True, platform="linux/" + image.getArch())) - logger.success(f"Image successfully updated") + logger.success(f"Image successfully installed") # Remove old image if not install_mode and image.isInstall() and UserConfig().auto_remove_images: cls.removeImage(image, upgrade_mode=not install_mode) From 9da26fd751fa1cd0a73b2a2b044976a7d1e6e496 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Mon, 18 Dec 2023 11:17:34 +0100 Subject: [PATCH 093/109] Fix install message --- exegol/utils/DockerUtils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index f684297a..4d9d6d35 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -457,7 +457,7 @@ def downloadImage(cls, image: ExegolImage, install_mode: bool = False) -> bool: stream=True, decode=True, platform="linux/" + image.getArch())) - logger.success(f"Image successfully installed") + logger.success(f"Image successfully {'installed' if install_mode else 'updated'}") # Remove old image if not install_mode and image.isInstall() and UserConfig().auto_remove_images: cls.removeImage(image, upgrade_mode=not install_mode) From 9ecb273de0cbc829d497c254173a15ec6992dc57 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Tue, 19 Dec 2023 15:04:04 +0100 Subject: [PATCH 094/109] Handle docker timeout errors --- exegol/utils/DockerUtils.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index 4d9d6d35..fe56edea 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -81,6 +81,9 @@ def listContainers(cls) -> List[ExegolContainer]: logger.critical(err.explanation) # Not reachable, critical logging will exit return # type: ignore + except TimeoutError: + logger.critical("Received a timeout error, Docker is busy... Unable to list containers, retry later.") + return # type: ignore for container in docker_containers: cls.__containers.append(ExegolContainer(container)) return cls.__containers @@ -217,6 +220,9 @@ def __loadDockerVolume(cls, volume_path: str, volume_name: str) -> Volume: logger.debug(e.explanation) else: raise NotFound('Volume must be reloaded') + except TimeoutError: + logger.error(f"Received a timeout error, Docker is busy... Volume {volume_name} cannot be automatically removed. Please, retry later the following command:{os.linesep}" + f" [orange3]docker volume rm {volume_name}[/orange3]") except NotFound: try: # Creating a docker volume bind to a host path @@ -231,9 +237,15 @@ def __loadDockerVolume(cls, volume_path: str, volume_name: str) -> Volume: logger.debug(err) logger.critical(err.explanation) return None # type: ignore + except TimeoutError: + logger.critical(f"Received a timeout error, Docker is busy... Volume {volume_name} cannot be created.") + return # type: ignore except APIError as err: logger.critical(f"Unexpected error by Docker SDK : {err}") return None # type: ignore + except TimeoutError: + logger.critical("Received a timeout error, Docker is busy... Unable to enumerate volume, retry later.") + return None # type: ignore return volume # # # Image Section # # # @@ -312,6 +324,9 @@ def getInstalledImage(cls, tag: str) -> ExegolImage: else: logger.critical(f"Error on image loading: {err}") return # type: ignore + except TimeoutError: + logger.critical("Received a timeout error, Docker is busy... Unable to list images, retry later.") + return # type: ignore return ExegolImage(docker_image=docker_local_image).autoLoad() else: for img in cls.__images: @@ -337,6 +352,9 @@ def __listLocalImages(cls, tag: Optional[str] = None) -> List[Image]: logger.critical(err.explanation) # Not reachable, critical logging will exit return # type: ignore + except TimeoutError: + logger.critical("Received a timeout error, Docker is busy... Unable to list local images, retry later.") + return # type: ignore # Filter out image non-related to the right repository result = [] ids = set() @@ -372,6 +390,9 @@ def __findLocalRecoveryImages(cls, include_untag: bool = False) -> List[Image]: except APIError as err: logger.debug(f"Error occurred in recovery mode: {err}") return [] + except TimeoutError: + logger.critical("Received a timeout error, Docker is busy... Unable to enumerate lost images, retry later.") + return # type: ignore result = [] id_list = set() for img in recovery_images: @@ -433,6 +454,9 @@ def __findImageMatch(cls, remote_image: ExegolImage): docker_image = cls.__client.images.get(f"{ConstantConfig.IMAGE_NAME}@{remote_id}") except ImageNotFound: raise ObjectNotFound + except TimeoutError: + logger.critical("Received a timeout error, Docker is busy... Unable to find a specific image, retry later.") + return # type: ignore remote_image.resetDockerImage() remote_image.setDockerObject(docker_image) @@ -471,6 +495,8 @@ def downloadImage(cls, image: ExegolImage, install_mode: bool = False) -> bool: else: logger.debug(f"Error: {err}") logger.critical(f"An error occurred while downloading this image: {err.explanation}") + except TimeoutError: + logger.critical(f"Received a timeout error, Docker is busy... Unable to download {name} image, retry later.") return False @classmethod @@ -492,6 +518,10 @@ def downloadVersionTag(cls, image: ExegolImage) -> Union[ExegolImage, str]: else: logger.debug(f"Error: {err}") return f"en unknown error occurred while downloading this image : {err.explanation}" + except TimeoutError: + logger.critical(f"Received a timeout error, Docker is busy... Unable to download an image tag, retry later the following command:{os.linesep}" + f" [orange3]docker pull --platform linux/{image.getArch()} {ConstantConfig.IMAGE_NAME}:{image.getLatestVersionName()}[/orange3].") + return # type: ignore @classmethod def removeImage(cls, image: ExegolImage, upgrade_mode: bool = False) -> bool: @@ -568,3 +598,6 @@ def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerf logger.debug(f"Error: {err}") else: logger.critical(f"An error occurred while building this image : {err}") + except TimeoutError: + logger.critical("Received a timeout error, Docker is busy... Unable to build the local image, retry later.") + return # type: ignore From 02bca4af2acb3c30e228c28a07fdb1a043e39129 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:18:48 +0100 Subject: [PATCH 095/109] Update bash scripts --- exegol/utils/imgsync/entrypoint.sh | 20 ++++++++++---------- exegol/utils/imgsync/spawn.sh | 11 ++++++----- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/exegol/utils/imgsync/entrypoint.sh b/exegol/utils/imgsync/entrypoint.sh index 06bc482c..d5a2262b 100755 --- a/exegol/utils/imgsync/entrypoint.sh +++ b/exegol/utils/imgsync/entrypoint.sh @@ -9,8 +9,8 @@ function exegol_init() { # Function specific function load_setups() { # Load custom setups (supported setups, and user setup) - [ -d /var/log/exegol ] || mkdir -p /var/log/exegol - if [[ ! -f /.exegol/.setup.lock ]]; then + [[ -d "/var/log/exegol" ]] || mkdir -p /var/log/exegol + if [[ ! -f "/.exegol/.setup.lock" ]]; then # Execute initial setup if lock file doesn't exist echo >/.exegol/.setup.lock # Run my-resources script. Logs starting with '[exegol]' will be print to the console and report back to the user through the wrapper. @@ -52,8 +52,8 @@ function shutdown() { # shellcheck disable=SC2046 kill $(pgrep -x -f -- -bash) 2>/dev/null # Wait for every active process to exit (e.g: shell logging compression, VPN closing, WebUI) - wait_list="$(pgrep -f "(.log|spawn.sh|vnc)" | grep -vE '^1$')" - for i in $wait_list; do + WAIT_LIST="$(pgrep -f "(.log|spawn.sh|vnc)" | grep -vE '^1$')" + for i in $WAIT_LIST; do # Waiting for: $i PID process to exit tail --pid="$i" -f /dev/null done @@ -62,12 +62,12 @@ function shutdown() { function _resolv_docker_host() { # On docker desktop host, resolving the host.docker.internal before starting a VPN connection for GUI applications - docker_ip=$(getent ahostsv4 host.docker.internal | head -n1 | awk '{ print $1 }') - if [ "$docker_ip" ]; then + DOCKER_IP=$(getent ahostsv4 host.docker.internal | head -n1 | awk '{ print $1 }') + if [[ "$DOCKER_IP" ]]; then # Add docker internal host resolution to the hosts file to preserve access to the X server - echo "$docker_ip host.docker.internal" >>/etc/hosts + echo "$DOCKER_IP host.docker.internal" >>/etc/hosts # If the container share the host networks, no need to add a static mapping - ip route list match "$docker_ip" table all | grep -v default || ip route add "$docker_ip/32" $(ip route list | grep default | head -n1 | grep -Eo '(via [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ )?dev [a-zA-Z0-9]+') || echo '[W]Exegol cannot add a static route to resolv your host X11 server. GUI applications may not work.' + ip route list match "$DOCKER_IP" table all | grep -v default || ip route add "$DOCKER_IP/32" $(ip route list | grep default | head -n1 | grep -Eo '(via [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+ )?dev [a-zA-Z0-9]+') || echo '[W]Exegol cannot add a static route to resolv your host X11 server. GUI applications may not work.' fi } @@ -116,8 +116,8 @@ exegol_init # Par each parameter for arg in "$@"; do # Check if the function exist - function_name=$(echo "$arg" | cut -d ' ' -f 1) - if declare -f "$function_name" > /dev/null; then + FUNCTION_NAME=$(echo "$arg" | cut -d ' ' -f 1) + if declare -f "$FUNCTION_NAME" > /dev/null; then $arg else echo "The function '$arg' doesn't exist." diff --git a/exegol/utils/imgsync/spawn.sh b/exegol/utils/imgsync/spawn.sh index 75649de4..f983ac59 100755 --- a/exegol/utils/imgsync/spawn.sh +++ b/exegol/utils/imgsync/spawn.sh @@ -8,11 +8,11 @@ function shell_logging() { # First parameter is the method to use for shell logging (default to script) - method=$1 + local method=$1 # The second parameter is the shell command to use for the user - user_shell=$2 + local user_shell=$2 # The third enable compression at the end of the session - compress=$3 + local compress=$3 # Test if the command is supported on the current image if ! command -v "$method" &> /dev/null @@ -26,6 +26,7 @@ function shell_logging() { umask 007 mkdir -p /workspace/logs/ + local filelog filelog="/workspace/logs/$(date +%d-%m-%Y_%H-%M-%S)_shell.${method}" case $method in @@ -45,8 +46,8 @@ function shell_logging() { ;; esac - if [ "$compress" = 'True' ]; then - echo 'Compressing logs, please wait...' + if [[ "$compress" = 'True' ]]; then + echo 'compressing logs, please wait...' gzip "$filelog" fi exit 0 From 6a0d2e64382cbae4aefd7ff6069aabe84b37681f Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:19:56 +0100 Subject: [PATCH 096/109] Update pypi publish method --- .github/workflows/entrypoint_nightly.yml | 5 ++++- .github/workflows/entrypoint_release.yml | 7 ++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/entrypoint_nightly.yml b/.github/workflows/entrypoint_nightly.yml index 9219d6ef..7c07ac77 100644 --- a/.github/workflows/entrypoint_nightly.yml +++ b/.github/workflows/entrypoint_nightly.yml @@ -15,6 +15,10 @@ jobs: build-n-publish: name: Build and publish Python 🐍 distributions to TestPyPI 📦 runs-on: ubuntu-latest + environment: nightly + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write needs: test steps: - uses: actions/checkout@master @@ -33,6 +37,5 @@ jobs: - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository-url: https://test.pypi.org/legacy/ skip-existing: true diff --git a/.github/workflows/entrypoint_release.yml b/.github/workflows/entrypoint_release.yml index 7cf0aa28..722cffd2 100644 --- a/.github/workflows/entrypoint_release.yml +++ b/.github/workflows/entrypoint_release.yml @@ -13,6 +13,10 @@ jobs: build-n-publish: name: Build and publish Python 🐍 distributions to PyPI 📦 runs-on: ubuntu-latest + environment: release + permissions: + # IMPORTANT: this permission is mandatory for trusted publishing + id-token: write needs: test steps: - uses: actions/checkout@master @@ -31,10 +35,7 @@ jobs: - name: Publish distribution 📦 to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: - password: ${{ secrets.TEST_PYPI_API_TOKEN }} repository-url: https://test.pypi.org/legacy/ skip-existing: true - name: Publish distribution 📦 to PyPI (prod) uses: pypa/gh-action-pypi-publish@release/v1 - with: - password: ${{ secrets.PYPI_API_TOKEN }} From 8dc87168562eae0d7be3a13b8742dc4e71df1e97 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:24:32 +0100 Subject: [PATCH 097/109] Remove rich bracket escape --- exegol/console/ExegolPrompt.py | 2 +- exegol/manager/UpdateManager.py | 4 ++-- exegol/model/ExegolContainer.py | 2 +- exegol/utils/ContainerLogStream.py | 2 +- exegol/utils/FsUtils.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/exegol/console/ExegolPrompt.py b/exegol/console/ExegolPrompt.py index c7c4fa8f..015e67f6 100644 --- a/exegol/console/ExegolPrompt.py +++ b/exegol/console/ExegolPrompt.py @@ -3,7 +3,7 @@ def Confirm(question: str, default: bool) -> bool: """Quick function to format rich Confirmation and options on every exegol interaction""" - default_text = "[bright_magenta][Y/n][/bright_magenta]" if default else "[bright_magenta]\[y/N][/bright_magenta]" + default_text = "[bright_magenta][Y/n][/bright_magenta]" if default else "[bright_magenta][y/N][/bright_magenta]" formatted_question = f"[bold blue][?][/bold blue] {question} {default_text}" return rich.prompt.Confirm.ask( formatted_question, diff --git a/exegol/manager/UpdateManager.py b/exegol/manager/UpdateManager.py index 79214d6e..f13699c5 100644 --- a/exegol/manager/UpdateManager.py +++ b/exegol/manager/UpdateManager.py @@ -263,7 +263,7 @@ def display_current_version(): if re.search(r'[a-z]', ConstantConfig.version, re.IGNORECASE): module = ExegolModules().getWrapperGit(fast_load=True) if module.isAvailable: - commit_version = f" [bright_black]\[{str(module.get_current_commit())[:8]}][/bright_black]" + commit_version = f" [bright_black][{str(module.get_current_commit())[:8]}][/bright_black]" return f"[blue]v{ConstantConfig.version}[/blue]{commit_version}" @classmethod @@ -290,7 +290,7 @@ def isUpdateTag(cls) -> bool: def display_latest_version(cls) -> str: last_version = DataCache().get_wrapper_data().last_version if len(last_version) == 8 and '.' not in last_version: - return f"[bright_black]\[{last_version}][/bright_black]" + return f"[bright_black][{last_version}][/bright_black]" return f"[blue]v{last_version}[/blue]" @classmethod diff --git a/exegol/model/ExegolContainer.py b/exegol/model/ExegolContainer.py index aedcaef4..57ac0eaa 100644 --- a/exegol/model/ExegolContainer.py +++ b/exegol/model/ExegolContainer.py @@ -132,7 +132,7 @@ def __start_container(self): logger.error(line) else: logger.verbose(line) - progress.update(status=f"[blue]\[Startup][/blue] {line}") + progress.update(status=f"[blue][Startup][/blue] {line}") except KeyboardInterrupt: # User can cancel startup logging with ctrl+C logger.warning("User skip startup status updates. Spawning a shell now.") diff --git a/exegol/utils/ContainerLogStream.py b/exegol/utils/ContainerLogStream.py index b862887a..51b278f8 100644 --- a/exegol/utils/ContainerLogStream.py +++ b/exegol/utils/ContainerLogStream.py @@ -62,7 +62,7 @@ def __next__(self): elif not self.__tips_sent and self.__until_date >= self.__tips_timedelta: self.__tips_sent = True logger.info("Your start-up sequence takes time, your my-resource setup configuration may be significant.") - logger.info("[orange3]\[Tips][/orange3] If you want to skip startup update, " + logger.info("[orange3][Tips][/orange3] If you want to skip startup update, " "you can use [green]CTRL+C[/green] and spawn a shell immediately. " "[blue](Startup sequence will continue in background)[/blue]") # Prepare the next iteration to fetch next logs diff --git a/exegol/utils/FsUtils.py b/exegol/utils/FsUtils.py index 5c70acff..2e1267c7 100644 --- a/exegol/utils/FsUtils.py +++ b/exegol/utils/FsUtils.py @@ -87,6 +87,6 @@ def setGidPermission(root_folder: Path): if perm_alert: logger.warning(f"In order to share files between your host and exegol (without changing the permission), you can run [orange3]manually[/orange3] this command from your [red]host[/red]:") logger.empty_line() - logger.raw(f"sudo chgrp -R $(id -g) {root_folder} && sudo find {root_folder} -type d -exec chmod g+rws {{}} \;", level=logging.WARNING) + logger.raw(f"sudo chgrp -R $(id -g) {root_folder} && sudo find {root_folder} -type d -exec chmod g+rws {{}} \\;", level=logging.WARNING) logger.empty_line() logger.empty_line() From 60e1cd636caae1af5b3f1def7dc88c5b645c073d Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Wed, 20 Dec 2023 00:43:55 +0100 Subject: [PATCH 098/109] New beta version --- exegol/config/ConstantConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index 2103c8eb..afe4895e 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -5,7 +5,7 @@ class ConstantConfig: """Constant parameters information""" # Exegol Version - version: str = "4.3.0b1" + version: str = "4.3.0b2" # Exegol documentation link documentation: str = "https://exegol.rtfd.io/" From 894cab294c2ccb30d815c04ee9f06265333bb8ac Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Thu, 21 Dec 2023 00:21:14 +0100 Subject: [PATCH 099/109] Fix pre-release pipeline --- .github/workflows/entrypoint_prerelease.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/entrypoint_prerelease.yml b/.github/workflows/entrypoint_prerelease.yml index ad9127f6..8eeaf985 100644 --- a/.github/workflows/entrypoint_prerelease.yml +++ b/.github/workflows/entrypoint_prerelease.yml @@ -14,8 +14,9 @@ jobs: uses: ./.github/workflows/sub_testing.yml preprod_test: - name: Code testing + name: Pre-prod code testing runs-on: ubuntu-latest + needs: code_test steps: - uses: actions/checkout@master with: @@ -30,7 +31,7 @@ jobs: build: name: Build Python 🐍 distributions runs-on: ubuntu-latest - needs: test + needs: preprod_test steps: - uses: actions/checkout@master with: From 37e5a329ca2a70dd92dddec0b2713d8a57193520 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Thu, 21 Dec 2023 00:36:36 +0100 Subject: [PATCH 100/109] Improve pre-release checks --- .github/workflows/entrypoint_prerelease.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/entrypoint_prerelease.yml b/.github/workflows/entrypoint_prerelease.yml index 8eeaf985..f8c21041 100644 --- a/.github/workflows/entrypoint_prerelease.yml +++ b/.github/workflows/entrypoint_prerelease.yml @@ -26,7 +26,11 @@ jobs: with: python-version: "3.10" - name: Find spawn.sh script version - run: egrep '^# Spawn Version:[0-9]+$' ./exegol/utils/imgsync/spawn.sh | cut -d ':' -f2 + run: egrep '^# Spawn Version:[0-9ab]+$' ./exegol/utils/imgsync/spawn.sh | cut -d ':' -f2 + - name: Check for prod readiness of spawn.sh script version + run: egrep '^# Spawn Version:[0-9]+$' ./exegol/utils/imgsync/spawn.sh + - name: Check package version (alpha and beta version cannot be released) + run: python3 -c 'from exegol.config.ConstantConfig import ConstantConfig; print(ConstantConfig.version); exit(any(c in ConstantConfig.version for c in ["a", "b"]))' build: name: Build Python 🐍 distributions From 900ff12e414140b95bdef12ef1f14b734ef0a206 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Thu, 21 Dec 2023 00:38:24 +0100 Subject: [PATCH 101/109] Change pre-release checks order --- .github/workflows/entrypoint_prerelease.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/entrypoint_prerelease.yml b/.github/workflows/entrypoint_prerelease.yml index f8c21041..1e201c0d 100644 --- a/.github/workflows/entrypoint_prerelease.yml +++ b/.github/workflows/entrypoint_prerelease.yml @@ -9,14 +9,9 @@ on: - "**.md" jobs: - code_test: - name: Python tests and checks - uses: ./.github/workflows/sub_testing.yml - preprod_test: name: Pre-prod code testing runs-on: ubuntu-latest - needs: code_test steps: - uses: actions/checkout@master with: @@ -32,10 +27,15 @@ jobs: - name: Check package version (alpha and beta version cannot be released) run: python3 -c 'from exegol.config.ConstantConfig import ConstantConfig; print(ConstantConfig.version); exit(any(c in ConstantConfig.version for c in ["a", "b"]))' + code_test: + name: Python tests and checks + needs: preprod_test + uses: ./.github/workflows/sub_testing.yml + build: name: Build Python 🐍 distributions runs-on: ubuntu-latest - needs: preprod_test + needs: code_test steps: - uses: actions/checkout@master with: From 3ab4065ac0a963f3f32c34becddf7b42adeecded Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Thu, 21 Dec 2023 01:00:21 +0100 Subject: [PATCH 102/109] Fix timeout error --- exegol/utils/DockerUtils.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index fe56edea..4f9e5089 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -81,7 +81,7 @@ def listContainers(cls) -> List[ExegolContainer]: logger.critical(err.explanation) # Not reachable, critical logging will exit return # type: ignore - except TimeoutError: + except ReadTimeout: logger.critical("Received a timeout error, Docker is busy... Unable to list containers, retry later.") return # type: ignore for container in docker_containers: @@ -220,7 +220,7 @@ def __loadDockerVolume(cls, volume_path: str, volume_name: str) -> Volume: logger.debug(e.explanation) else: raise NotFound('Volume must be reloaded') - except TimeoutError: + except ReadTimeout: logger.error(f"Received a timeout error, Docker is busy... Volume {volume_name} cannot be automatically removed. Please, retry later the following command:{os.linesep}" f" [orange3]docker volume rm {volume_name}[/orange3]") except NotFound: @@ -237,13 +237,13 @@ def __loadDockerVolume(cls, volume_path: str, volume_name: str) -> Volume: logger.debug(err) logger.critical(err.explanation) return None # type: ignore - except TimeoutError: + except ReadTimeout: logger.critical(f"Received a timeout error, Docker is busy... Volume {volume_name} cannot be created.") return # type: ignore except APIError as err: logger.critical(f"Unexpected error by Docker SDK : {err}") return None # type: ignore - except TimeoutError: + except ReadTimeout: logger.critical("Received a timeout error, Docker is busy... Unable to enumerate volume, retry later.") return None # type: ignore return volume @@ -324,7 +324,7 @@ def getInstalledImage(cls, tag: str) -> ExegolImage: else: logger.critical(f"Error on image loading: {err}") return # type: ignore - except TimeoutError: + except ReadTimeout: logger.critical("Received a timeout error, Docker is busy... Unable to list images, retry later.") return # type: ignore return ExegolImage(docker_image=docker_local_image).autoLoad() @@ -352,7 +352,7 @@ def __listLocalImages(cls, tag: Optional[str] = None) -> List[Image]: logger.critical(err.explanation) # Not reachable, critical logging will exit return # type: ignore - except TimeoutError: + except ReadTimeout: logger.critical("Received a timeout error, Docker is busy... Unable to list local images, retry later.") return # type: ignore # Filter out image non-related to the right repository @@ -390,7 +390,7 @@ def __findLocalRecoveryImages(cls, include_untag: bool = False) -> List[Image]: except APIError as err: logger.debug(f"Error occurred in recovery mode: {err}") return [] - except TimeoutError: + except ReadTimeout: logger.critical("Received a timeout error, Docker is busy... Unable to enumerate lost images, retry later.") return # type: ignore result = [] @@ -454,7 +454,7 @@ def __findImageMatch(cls, remote_image: ExegolImage): docker_image = cls.__client.images.get(f"{ConstantConfig.IMAGE_NAME}@{remote_id}") except ImageNotFound: raise ObjectNotFound - except TimeoutError: + except ReadTimeout: logger.critical("Received a timeout error, Docker is busy... Unable to find a specific image, retry later.") return # type: ignore remote_image.resetDockerImage() @@ -495,7 +495,7 @@ def downloadImage(cls, image: ExegolImage, install_mode: bool = False) -> bool: else: logger.debug(f"Error: {err}") logger.critical(f"An error occurred while downloading this image: {err.explanation}") - except TimeoutError: + except ReadTimeout: logger.critical(f"Received a timeout error, Docker is busy... Unable to download {name} image, retry later.") return False @@ -518,7 +518,7 @@ def downloadVersionTag(cls, image: ExegolImage) -> Union[ExegolImage, str]: else: logger.debug(f"Error: {err}") return f"en unknown error occurred while downloading this image : {err.explanation}" - except TimeoutError: + except ReadTimeout: logger.critical(f"Received a timeout error, Docker is busy... Unable to download an image tag, retry later the following command:{os.linesep}" f" [orange3]docker pull --platform linux/{image.getArch()} {ConstantConfig.IMAGE_NAME}:{image.getLatestVersionName()}[/orange3].") return # type: ignore @@ -598,6 +598,6 @@ def buildImage(cls, tag: str, build_profile: Optional[str] = None, build_dockerf logger.debug(f"Error: {err}") else: logger.critical(f"An error occurred while building this image : {err}") - except TimeoutError: + except ReadTimeout: logger.critical("Received a timeout error, Docker is busy... Unable to build the local image, retry later.") return # type: ignore From 0e51d85aa28ca76cd16c77ab6f9a3d4caf80461a Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Thu, 21 Dec 2023 16:00:43 +0100 Subject: [PATCH 103/109] TODO & config update --- exegol/config/ConstantConfig.py | 2 +- exegol/config/UserConfig.py | 7 +++---- exegol/model/ContainerConfig.py | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index afe4895e..5c07c013 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -42,7 +42,7 @@ def findResourceContextPath(cls, resource_folder: str, source_path: str) -> Path Support source clone installation and pip package (venv / user / global context)""" local_src = cls.src_root_path_obj / source_path if local_src.is_dir() or local_src.is_file(): - # If exegol is clone from github, build context is accessible from root src + # If exegol is clone from GitHub, build context is accessible from root src return local_src else: # If install from pip diff --git a/exegol/config/UserConfig.py b/exegol/config/UserConfig.py index f0c8a3ca..c32b873d 100644 --- a/exegol/config/UserConfig.py +++ b/exegol/config/UserConfig.py @@ -71,20 +71,19 @@ def _build_file_content(self): # Configure your Exegol Desktop desktop: - # Enables the desktop mode all the time - # If this attribute is set to True, then using the CLI --desktop option will be inverted and will DISABLE the desktop + # Enables or not the desktop mode by default + # If this attribute is set to True, then using the CLI --desktop option will be inverted and will DISABLE the feature enabled_by_default: {self.desktop_default_enable} # Default desktop protocol,can be "http", or "vnc" (additional protocols to come in the future, check online documentation for updates). default_protocol: {self.desktop_default_proto} - # Desktop service is exposed on localhost by default. If set to true, services will be exposed on localhost (127.0.0.1) other it will be exposed on 0.0.0.0. This setting can be overwritten with --desktop-config + # Desktop service is exposed on localhost by default. If set to true, services will be exposed on localhost (127.0.0.1) otherwise it will be exposed on 0.0.0.0. This setting can be overwritten with --desktop-config localhost_by_default: {self.desktop_default_localhost} """ # TODO handle default image selection # TODO handle default start container - # TODO add custom build profiles path return config @staticmethod diff --git a/exegol/model/ContainerConfig.py b/exegol/model/ContainerConfig.py index 79fd6502..154fef5f 100644 --- a/exegol/model/ContainerConfig.py +++ b/exegol/model/ContainerConfig.py @@ -505,7 +505,6 @@ def enableDesktop(self, desktop_config: str = ""): self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__desktop_port)) else: # If we do not specify the host to the container it will automatically choose eth0 interface - # TODO ensure there is an eth0 interface # Using default port for the service self.addEnv(self.ExegolEnv.desktop_port.value, str(self.__default_desktop_port.get(self.__desktop_proto))) # Exposing desktop service From 0a3ef75439fffccc755ae532ac23d86ba7017026 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Thu, 21 Dec 2023 16:00:54 +0100 Subject: [PATCH 104/109] Ready for new release --- exegol-docker-build | 2 +- exegol-resources | 2 +- exegol/config/ConstantConfig.py | 2 +- exegol/utils/imgsync/spawn.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/exegol-docker-build b/exegol-docker-build index ad28264b..0fa6ea0f 160000 --- a/exegol-docker-build +++ b/exegol-docker-build @@ -1 +1 @@ -Subproject commit ad28264bd1698f45d522866d0033fb53942dbd98 +Subproject commit 0fa6ea0f4116434b12a72697f9459b3093a53cad diff --git a/exegol-resources b/exegol-resources index b36c094f..fd97ffe9 160000 --- a/exegol-resources +++ b/exegol-resources @@ -1 +1 @@ -Subproject commit b36c094f3b38d18a40d707090d2213304f231794 +Subproject commit fd97ffe9b10fd18cf13f57864e8a04fc68c0e43a diff --git a/exegol/config/ConstantConfig.py b/exegol/config/ConstantConfig.py index 5c07c013..6742d634 100644 --- a/exegol/config/ConstantConfig.py +++ b/exegol/config/ConstantConfig.py @@ -5,7 +5,7 @@ class ConstantConfig: """Constant parameters information""" # Exegol Version - version: str = "4.3.0b2" + version: str = "4.3.0" # Exegol documentation link documentation: str = "https://exegol.rtfd.io/" diff --git a/exegol/utils/imgsync/spawn.sh b/exegol/utils/imgsync/spawn.sh index f983ac59..6283b390 100755 --- a/exegol/utils/imgsync/spawn.sh +++ b/exegol/utils/imgsync/spawn.sh @@ -1,7 +1,7 @@ #!/bin/bash # DO NOT CHANGE the syntax or text of the following line, only increment the version number -# Spawn Version:2b +# Spawn Version:2 # The spawn version allow the wrapper to compare the current version of the spawn.sh inside the container compare to the one on the current wrapper version. # On new container, this file is automatically updated through a docker volume # For legacy container, this version is fetch and the file updated if needed. From 1fa1f2a1e5a22edc784a7fa23f19ea1db290f2f2 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Thu, 21 Dec 2023 18:07:40 +0100 Subject: [PATCH 105/109] Fix empty string on build date --- exegol/model/ExegolImage.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index c91cc080..5bb3c1d3 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -589,7 +589,10 @@ def getEntrypointConfig(self) -> Optional[Union[str, List[str]]]: def getBuildDate(self): """Build date getter""" - if "N/A" not in self.__build_date.upper(): + if not self.__build_date: + # Handle empty string + return "[bright_black]N/A[/bright_black]" + elif "N/A" not in self.__build_date.upper(): return datetime.strptime(self.__build_date, "%Y-%m-%dT%H:%M:%SZ").strftime("%d/%m/%Y %H:%M") else: return self.__build_date From e94d4f3727bba014ed6f958d9574deb3a51cb680 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Fri, 22 Dec 2023 15:47:37 +0100 Subject: [PATCH 106/109] Update submodule ref --- exegol-docker-build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exegol-docker-build b/exegol-docker-build index 0fa6ea0f..ef1bc9cc 160000 --- a/exegol-docker-build +++ b/exegol-docker-build @@ -1 +1 @@ -Subproject commit 0fa6ea0f4116434b12a72697f9459b3093a53cad +Subproject commit ef1bc9cc98632a3bc7a02c97a9f7855f4bccbde4 From e330e665184ef5bdd59a8ea533fc5537a3e118f2 Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Fri, 22 Dec 2023 15:51:18 +0100 Subject: [PATCH 107/109] Align test release version --- tests/test_exegol.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_exegol.py b/tests/test_exegol.py index 442eb5b8..391312ae 100644 --- a/tests/test_exegol.py +++ b/tests/test_exegol.py @@ -2,4 +2,4 @@ def test_version(): - assert __version__ == '4.2.6' + assert __version__ == '4.3.0' From f701cad03db956107a26a5b294ed7a06e5329e1b Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Fri, 22 Dec 2023 16:03:15 +0100 Subject: [PATCH 108/109] Upgrade dependencies versions --- requirements.txt | 10 +++++----- setup.py | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/requirements.txt b/requirements.txt index bc41ff5f..26970b65 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ -docker~=6.1.3 +docker~=7.0.0 requests>=2.31.0 -rich~=13.4.2 -GitPython~=3.1.29 -PyYAML>=6.0 -argcomplete~=3.1.1 \ No newline at end of file +rich~=13.7.0 +GitPython~=3.1.40 +PyYAML>=6.0.1 +argcomplete~=3.2.1 \ No newline at end of file diff --git a/setup.py b/setup.py index 4e48b95e..96437287 100644 --- a/setup.py +++ b/setup.py @@ -54,12 +54,12 @@ "Operating System :: OS Independent", ], install_requires=[ - 'docker~=6.1.3', + 'docker~=7.0.0', 'requests>=2.31.0', - 'rich~=13.4.2', + 'rich~=13.7.0', 'PyYAML', - 'GitPython', - 'argcomplete~=3.1.1' + 'GitPython~=3.1.40', + 'argcomplete~=3.2.1' ], packages=find_packages(exclude=["tests"]), include_package_data=True, From d82056eb1c19c29459f68aadbc1d58f503c0bcef Mon Sep 17 00:00:00 2001 From: Dramelac <dramelac@users.noreply.github.com> Date: Fri, 22 Dec 2023 16:06:51 +0100 Subject: [PATCH 109/109] Add py3.12 tests --- .github/workflows/sub_testing.yml | 6 +++--- setup.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sub_testing.yml b/.github/workflows/sub_testing.yml index 56e678ac..de5cc5a5 100644 --- a/.github/workflows/sub_testing.yml +++ b/.github/workflows/sub_testing.yml @@ -14,7 +14,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Install requirements run: python -m pip install --user mypy types-requests types-PyYAML - name: Run code analysis (package) @@ -31,7 +31,7 @@ jobs: strategy: fail-fast: false matrix: - version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] os: [win32, linux, darwin] steps: - uses: actions/checkout@master @@ -40,7 +40,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Install requirements run: python -m pip install --user mypy types-requests types-PyYAML - name: Check python compatibility for ${{ matrix.os }}/${{ matrix.version }} diff --git a/setup.py b/setup.py index 96437287..4bc34d53 100644 --- a/setup.py +++ b/setup.py @@ -50,6 +50,7 @@ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", ],