From 8c100dcc7d0b7f623d301d3f8c48cb7384bb8395 Mon Sep 17 00:00:00 2001 From: Dramelac Date: Sat, 9 Nov 2024 14:13:18 +0100 Subject: [PATCH] Improve image loader wait time --- exegol/model/ExegolImage.py | 13 ++++++--- exegol/utils/DockerUtils.py | 53 +++++++++++++++++++++---------------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/exegol/model/ExegolImage.py b/exegol/model/ExegolImage.py index 49479e4b..596b88d3 100644 --- a/exegol/model/ExegolImage.py +++ b/exegol/model/ExegolImage.py @@ -3,6 +3,7 @@ from docker.models.containers import Container from docker.models.images import Image +from rich.status import Status from exegol.config.DataCache import DataCache from exegol.console import ConsoleFormat @@ -179,7 +180,7 @@ def setDockerObject(self, docker_image: Image): # backup plan: Use label to retrieve image version self.__labelVersionParsing() - def setMetaImage(self, meta: MetaImages): + def setMetaImage(self, meta: MetaImages, status: Status): dockerhub_data = meta.getDockerhubImageForArch(self.getArch()) self.__is_remote = True if self.__version_specific: @@ -189,6 +190,8 @@ def setMetaImage(self, meta: MetaImages): self.__outdated = self.__version_specific elif not meta.version: # nightly image don't have a version in their tag. The latest version must be fetch from label on the registry directly + logger.verbose(f"Fetch latest [green]{self.getName()}[/green] image version") + status.update(status=f"Retrieving latest [green]{self.getName()}[/green] image version") fetch_version = WebUtils.getRemoteVersion(self.__name) if fetch_version: meta.version = fetch_version @@ -269,7 +272,7 @@ def autoLoad(self, from_cache: bool = True) -> 'ExegolImage': logger.debug(f"Auto-load remote version for the specific image '{self.__name}'") # Find remote metadata for the specific current image with console.status(f"Synchronization of the [green]{self.__name}[/green] image status...", - spinner_style="blue"): + spinner_style="blue") as s: remote_digest = None version = None if from_cache: @@ -283,6 +286,8 @@ def autoLoad(self, from_cache: bool = True) -> 'ExegolImage': break if not from_cache or version is None: remote_digest = WebUtils.getMetaDigestId(self.__name) + logger.verbose(f"Fetch latest [green]{self.getName()}[/green] image version") + s.update(status=f"Retrieving latest [green]{self.getName()}[/green] image version") version = WebUtils.getRemoteVersion(self.__name) if remote_digest is not None: self.__setLatestRemoteId(remote_digest) @@ -347,7 +352,7 @@ def __mergeMetaImages(cls, images: List[MetaImages]): pass @classmethod - def mergeImages(cls, remote_images: List[MetaImages], local_images: List[Image]) -> List['ExegolImage']: + def mergeImages(cls, remote_images: List[MetaImages], local_images: List[Image], status: Status) -> List['ExegolImage']: """Compare and merge local images and remote images. Use case to process : - up-to-date : "Version specific" image can use exact digest_id matching. Latest image must match corresponding tag @@ -400,7 +405,7 @@ def mergeImages(cls, remote_images: List[MetaImages], local_images: List[Image]) selected = default if selected: # Remote match found - current_local_img.setMetaImage(selected) + current_local_img.setMetaImage(selected, status) else: if len(remote_images) > 0: # If there is some result from internet but no match and this is not a local image, this image is probably discontinued or the remote image is too old (relative to other images) diff --git a/exegol/utils/DockerUtils.py b/exegol/utils/DockerUtils.py index b0ed4e60..11000c56 100644 --- a/exegol/utils/DockerUtils.py +++ b/exegol/utils/DockerUtils.py @@ -10,6 +10,7 @@ from docker.models.images import Image from docker.models.volumes import Volume from requests import ReadTimeout +from rich.status import Status from exegol.config.ConstantConfig import ConstantConfig from exegol.config.DataCache import DataCache @@ -258,9 +259,14 @@ def listImages(self, include_version_tag: bool = False, include_locked: bool = F """List available docker images. Return a list of ExegolImage""" if self.__images is None: - remote_images = self.__listRemoteImages() - local_images = self.__listLocalImages() - self.__images = ExegolImage.mergeImages(remote_images, local_images) + logger.verbose("Loading every Exegol images") + with console.status(f"Loading Exegol images from registry", spinner_style="blue") as s: + remote_images = self.__listRemoteImages(s) + logger.verbose("Retrieve [green]local[/green] Exegol images") + s.update(status=f"Retrieving [green]local[/green] Exegol images") + local_images = self.__listLocalImages() + self.__images = ExegolImage.mergeImages(remote_images, local_images, s) + logger.verbose("Images fetched") result = self.__images assert result is not None # Caching latest images @@ -405,7 +411,8 @@ def __findLocalRecoveryImages(self, include_untag: bool = False) -> List[Image]: id_list.add(img.id) return result - def __listRemoteImages(self) -> List[MetaImages]: + @staticmethod + def __listRemoteImages(status: Status) -> List[MetaImages]: """List remote dockerhub images available. Return a list of ExegolImage""" logger.debug("Fetching remote image tags, digests and sizes") @@ -416,25 +423,25 @@ def __listRemoteImages(self) -> List[MetaImages]: current_page = 1 url: Optional[str] = f"https://{ConstantConfig.DOCKER_HUB}/v2/repositories/{ConstantConfig.IMAGE_NAME}/tags?page=1&page_size={page_size}" # Handle multi-page tags from registry - with console.status(f"Loading registry information from [green]{url}[/green]", spinner_style="blue") as s: - while url is not None: - if current_page > page_max: - logger.debug("Max page limit reached. In non-verbose mode, downloads will stop there.") - if not logger.isEnabledFor(ExeLog.VERBOSE): - break - current_page += 1 - s.update(status=f"Fetching registry information from [green]{url}[/green]") - docker_repo_response = WebUtils.runJsonRequest(url, "Dockerhub") - if docker_repo_response is None: - logger.warning("Skipping online queries.") - return [] - error_message = docker_repo_response.get("message") - if error_message: - logger.error(f"Dockerhub send an error message: {error_message}") - for docker_images in docker_repo_response.get("results", []): - meta_image = MetaImages(docker_images) - remote_results.append(meta_image) - url = docker_repo_response.get("next") # handle multiple page tags + while url is not None: + if current_page > page_max: + logger.debug("Max page limit reached. In non-verbose mode, downloads will stop there.") + if not logger.isEnabledFor(ExeLog.VERBOSE): + break + current_page += 1 + if logger.isEnabledFor(ExeLog.VERBOSE): + status.update(status=f"Fetching registry information from [green]{url}[/green]") + docker_repo_response = WebUtils.runJsonRequest(url, "Dockerhub") + if docker_repo_response is None: + logger.warning("Skipping online queries.") + return [] + error_message = docker_repo_response.get("message") + if error_message: + logger.error(f"Dockerhub send an error message: {error_message}") + for docker_images in docker_repo_response.get("results", []): + meta_image = MetaImages(docker_images) + remote_results.append(meta_image) + url = docker_repo_response.get("next") # handle multiple page tags # Remove duplication (version specific / latest release) return remote_results