From 6dbd430b4af29ed7bd634fa2c16c574a4b10b1c3 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 17 Aug 2022 15:10:48 -0400 Subject: [PATCH] Make check_exists cache negative results as well as positive ones When loading a workflow with URL references that can't be resolved, don't keep trying them over and over again. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- schema_salad/fetcher.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/schema_salad/fetcher.py b/schema_salad/fetcher.py index 8ff21810..3bca793d 100644 --- a/schema_salad/fetcher.py +++ b/schema_salad/fetcher.py @@ -105,7 +105,10 @@ def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str raise ValidationException(f"Unsupported scheme in url: {url}") def check_exists(self, url: str) -> bool: - if url in self.cache: + entry = self.cache.get(url) + if entry is False: + return False + elif entry is not None: return True split = urllib.parse.urlsplit(url) @@ -118,6 +121,7 @@ def check_exists(self, url: str) -> bool: resp = self.session.head(url, allow_redirects=True) resp.raise_for_status() except Exception: + self.cache[url] = False return False self.cache[url] = True return True