From 032258766af553813fc616b4b4ee808aabe77f4f Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Thu, 29 Feb 2024 17:25:47 +0000 Subject: [PATCH] fix: Accept SVG as avatar_url (#2083) Make download_avatar task to accept .svg up to 3MB Issue: AAH-2836 --- CHANGES/2836.bugfix | 1 + .../commands/download-namespace-logos.py | 52 ++++++++++++++++++- galaxy_ng/app/tasks/namespaces.py | 38 ++++++++++---- .../api/test_namespace_management.py | 1 + 4 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 CHANGES/2836.bugfix diff --git a/CHANGES/2836.bugfix b/CHANGES/2836.bugfix new file mode 100644 index 0000000000..ab10cfb81c --- /dev/null +++ b/CHANGES/2836.bugfix @@ -0,0 +1 @@ +Support SVG avatar image on namespaces diff --git a/galaxy_ng/app/management/commands/download-namespace-logos.py b/galaxy_ng/app/management/commands/download-namespace-logos.py index 9c5c200522..6bf64044b0 100644 --- a/galaxy_ng/app/management/commands/download-namespace-logos.py +++ b/galaxy_ng/app/management/commands/download-namespace-logos.py @@ -24,15 +24,51 @@ class Command(BaseCommand): def add_arguments(self, parser): parser.add_argument("--namespace", help="find and sync only this namespace name") + parser.add_argument( + "--sha-report", + default=False, + action="store_true", + required=False, + help="report the number of namespaces with avatar_url but missing avatar_sha256", + dest="sha_report", + ) + parser.add_argument( + "--only-missing-sha", + default=False, + action="store_true", + required=False, + help="When set it will limit the logo download only to those namespaces missing sha", + dest="only_missing_sha", + ) def echo(self, message, style=None): style = style or self.style.SUCCESS self.stdout.write(style(message)) def handle(self, *args, **options): + # all namespaces having avatar_url must have avatar_sha256 set + # query for namespaces missing avatar_sha256 + ns_missing_avatar_sha = Namespace.objects.filter( + _avatar_url__isnull=False, + last_created_pulp_metadata__avatar_sha256__isnull=True + ) + if ns_missing_avatar_sha: + self.echo( + f"{ns_missing_avatar_sha.count()} Namespaces missing avatar_sha256", + self.style.ERROR + ) + self.echo(", ".join(ns_missing_avatar_sha.values_list("name", flat=True))) + else: + self.echo("There are no namespaces missing avatar_sha256!") + + if options["sha_report"]: # --sha-report indicated only report was requested + return + + self.echo("Proceeding with namespace logo downloads") kwargs = { 'namespace_name': options['namespace'], + 'only_missing_sha': options['only_missing_sha'], } task = dispatch( @@ -52,11 +88,23 @@ def handle(self, *args, **options): sys.exit(1) -def download_all_logos(namespace_name=None): +def download_all_logos(namespace_name=None, only_missing_sha=False): + """Force logo downloads. + namespace: limit to specified namespace (or list of namespaces) + only_missing_sha: Limit to namespaces having avatar_url but missing avatar_sha256 + """ if namespace_name: - qs = Namespace.objects.filter(name=namespace_name) + namespaces = namespace_name.split(",") + qs = Namespace.objects.filter(name__in=namespaces) else: qs = Namespace.objects.all() + + if only_missing_sha: + qs = qs.filter( + _avatar_url__isnull=False, + last_created_pulp_metadata__avatar_sha256__isnull=True + ) + for namespace in qs: download_logo = False if namespace._avatar_url: diff --git a/galaxy_ng/app/tasks/namespaces.py b/galaxy_ng/app/tasks/namespaces.py index ba4a9d821d..24b0a116a5 100644 --- a/galaxy_ng/app/tasks/namespaces.py +++ b/galaxy_ng/app/tasks/namespaces.py @@ -1,5 +1,7 @@ import aiohttp import asyncio +import contextlib +import xml.etree.cElementTree as et from django.db import transaction from django.forms.fields import ImageField @@ -15,6 +17,9 @@ from galaxy_ng.app.models import Namespace +MAX_AVATAR_SIZE = 3 * 1024 * 1024 # 3MB + + def dispatch_create_pulp_namespace_metadata(galaxy_ns, download_logo): dispatch( @@ -26,11 +31,16 @@ def dispatch_create_pulp_namespace_metadata(galaxy_ns, download_logo): ) -def _download_avatar(url): +def _download_avatar(url, namespace_name): + # User-Agent needs to be added to avoid timing out on throtled servers. + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0)' # + + ' Gecko/20100101 Firefox/71.0' + } timeout = aiohttp.ClientTimeout(total=None, sock_connect=600, sock_read=600) conn = aiohttp.TCPConnector(force_close=True) session = aiohttp.ClientSession( - connector=conn, timeout=timeout, headers=None, requote_redirect_url=False + connector=conn, timeout=timeout, headers=headers, requote_redirect_url=False ) try: @@ -41,19 +51,29 @@ def _download_avatar(url): finally: asyncio.get_event_loop().run_until_complete(session.close()) - try: + # Limit size of the avatar to avoid memory issues when validating it + if img.artifact_attributes["size"] > MAX_AVATAR_SIZE: + raise ValidationError( + f"Avatar for {namespace_name} on {url} larger than {MAX_AVATAR_SIZE / 1024 / 1024}MB" + ) + + with contextlib.suppress(Artifact.DoesNotExist): return Artifact.objects.get(sha256=img.artifact_attributes["sha256"]) - except Artifact.DoesNotExist: - pass with open(img.path, "rb") as f: tf = PulpTemporaryUploadedFile.from_file(f) - try: ImageField().to_python(tf) except ValidationError: - print("file is not an image") - return + # Not a PIL valid image lets handle SVG case + tag = None + with contextlib.suppress(et.ParseError): + f.seek(0) + tag = et.parse(f).find(".").tag + if tag != '{http://www.w3.org/2000/svg}svg': + raise ValidationError( + f"Provided avatar_url for {namespace_name} on {url} is not a valid image" + ) # the artifact has to be saved before the file is closed, or s3transfer # will throw an error. @@ -71,7 +91,7 @@ def _create_pulp_namespace(galaxy_ns_pk, download_logo): avatar_artifact = None if download_logo: - avatar_artifact = _download_avatar(galaxy_ns._avatar_url) + avatar_artifact = _download_avatar(galaxy_ns._avatar_url, galaxy_ns.name) avatar_sha = None if avatar_artifact: diff --git a/galaxy_ng/tests/integration/api/test_namespace_management.py b/galaxy_ng/tests/integration/api/test_namespace_management.py index 5ded39e48c..50782b9be2 100644 --- a/galaxy_ng/tests/integration/api/test_namespace_management.py +++ b/galaxy_ng/tests/integration/api/test_namespace_management.py @@ -177,6 +177,7 @@ def test_namespace_edit_logo(galaxy_client): wait_for_all_tasks_gk(gc) updated_again_namespace = gc.get(f"_ui/v1/my-namespaces/{name}/") assert updated_namespace["avatar_url"] != updated_again_namespace["avatar_url"] + assert updated_namespace["avatar_sha256"] is not None # verify no additional namespaces are created resp = gc.get("_ui/v1/my-namespaces/")