From 31d0117430c6d071eaa08ca326c6ffa78a170f23 Mon Sep 17 00:00:00 2001 From: Peter Amstutz Date: Wed, 17 Aug 2022 15:10:48 -0400 Subject: [PATCH] Make check_exists cache negative results as well as positive ones When loading a workflow with URL references that can't be resolved, don't keep trying them over and over again. Arvados-DCO-1.1-Signed-off-by: Peter Amstutz --- schema_salad/fetcher.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/schema_salad/fetcher.py b/schema_salad/fetcher.py index 1759758d..6720235e 100644 --- a/schema_salad/fetcher.py +++ b/schema_salad/fetcher.py @@ -108,7 +108,10 @@ def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str raise ValidationException(f"Unsupported scheme in url: {url}") def check_exists(self, url: str) -> bool: - if url in self.cache: + entry = self.cache.get(url) + if entry is False: + return False + elif entry is not None: return True split = urllib.parse.urlsplit(url) @@ -121,6 +124,7 @@ def check_exists(self, url: str) -> bool: resp = self.session.head(url, allow_redirects=True) resp.raise_for_status() except Exception: + self.cache[url] = False return False self.cache[url] = True return True