From 0d5469e76f08a0df6f146965e4a1a7a41cb5a949 Mon Sep 17 00:00:00 2001 From: Jeffrey Kinard Date: Fri, 27 Oct 2023 13:59:38 -0400 Subject: [PATCH] [YAML] add GCS support for external transform jars Signed-off-by: Jeffrey Kinard --- sdks/python/apache_beam/utils/subprocess_server.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/utils/subprocess_server.py b/sdks/python/apache_beam/utils/subprocess_server.py index f566c3ea2914..f6e214046f37 100644 --- a/sdks/python/apache_beam/utils/subprocess_server.py +++ b/sdks/python/apache_beam/utils/subprocess_server.py @@ -36,6 +36,7 @@ import grpc +from apache_beam.io.filesystems import FileSystems from apache_beam.version import __version__ as beam_version _LOGGER = logging.getLogger(__name__) @@ -272,7 +273,10 @@ def local_jar(cls, url, cache_dir=None): os.makedirs(cache_dir) # TODO: Clean up this cache according to some policy. try: - url_read = urlopen(url) + try: + url_read = FileSystems.open(url) + except ValueError: + url_read = urlopen(url) with open(cached_jar + '.tmp', 'wb') as jar_write: shutil.copyfileobj(url_read, jar_write, length=1 << 20) os.rename(cached_jar + '.tmp', cached_jar)