From 17276aa50a35820d192a05ed747edde1abe4f1e0 Mon Sep 17 00:00:00 2001 From: wachsylon Date: Mon, 17 Jul 2023 17:27:43 +0200 Subject: [PATCH 1/5] Added temporary url support_ --- swiftspec/core.py | 34 ++++++++++++++++++++++++++++------ 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/swiftspec/core.py b/swiftspec/core.py index 82fd5ec..2e4bb8a 100644 --- a/swiftspec/core.py +++ b/swiftspec/core.py @@ -142,18 +142,34 @@ def get_tokens_from_env(self): token = os.environ.get("OS_AUTH_TOKEN") url = os.environ.get("OS_STORAGE_URL") if token and url: - return [{"token": token, "url": url}] + return [{"X-Auth-Token": token, "url": url}] + + sig = os.environ.get("TEMP_URL_SIG") + expire = os.environ.get("TEMP_URL_EXPIRES") + prf = os.environ.get("TEMP_URL_PREFIX") + + if url and sig and expire: + return [{"url":url,"temp_url_sig": sig, "temp_url_expires": expire, "temp_url_prefix":prf}] else: return [] def headers_for_url(self, url): headers = {} for auth in self.auth: - if url.startswith(auth["url"]): - headers["X-Auth-Token"] = auth["token"] + if url.startswith(auth["url"]) and "X-Auth-Token" in auth: + headers["X-Auth-Token"]=auth["X-Auth-Token"] break return headers + def params_for_url(self, url): + params = {} + for auth in self.auth: + if url.startswith(auth["url"]) and "temp_url_sig" in auth: + params=auth.copy() + del params["url"] + break + return params + @classmethod def _strip_protocol(cls, path): """For SWIFT, we always want to keep the full URL""" @@ -166,6 +182,7 @@ async def _ls(self, path, detail=True, **kwargs): params = { "format": "json", } + params.update(params_for_url(url)) url = f"https://{ref.host}/v1/{ref.account}" async with session.get( url, params=params, headers=self.headers_for_url(url) @@ -192,6 +209,8 @@ async def _ls(self, path, detail=True, **kwargs): "delimiter": "/", "prefix": prefix, } + params.update(params_for_url(url)) + url = f"https://{ref.host}/v1/{ref.account}/{ref.container}" async with session.get( url, params=params, headers=self.headers_for_url(url) @@ -222,6 +241,7 @@ def _raise_not_found_for_status(self, response, ref): async def _cat_file(self, path, start=None, end=None, **kwargs): ref = SWIFTRef(path) headers = self.headers_for_url(ref.http_url) + params = self.params_for_url(ref.http_url) if start is not None: assert start >= 0 if end is not None: @@ -235,7 +255,7 @@ async def _cat_file(self, path, start=None, end=None, **kwargs): headers["Range"] = f"bytes=0-{end}" session = await self.set_session() - async with session.get(ref.http_url, headers=headers) as res: + async with session.get(ref.http_url, params=params, headers=headers) as res: self._raise_not_found_for_status(res, ref) return await res.read() @@ -251,13 +271,14 @@ async def _pipe_file(self, path, data, chunksize=50 * 2**20, **kwargs): url = ref.http_url headers = self.headers_for_url(url) + params = self.params_for_url(url) headers["Content-Length"] = str(size) if self.verify_uploads: # in swift, ETag is alwas the MD5sum and will be used by the server to verify the upload headers["ETag"] = md5(data).hexdigest() session = await self.set_session() - async with session.put(url, data=data, headers=headers) as res: + async with session.put(url, data=data, params=params, headers=headers) as res: res.raise_for_status() async def _rm_file(self, path, missing_is_ok=False, **kwargs): @@ -325,8 +346,9 @@ async def _info(self, path, **kwargs): "size": None, } headers = self.headers_for_url(ref.http_url) + params = self.params_for_url(ref.http_url) session = await self.set_session() - async with session.head(ref.http_url, headers=headers) as res: + async with session.head(ref.http_url, params=params, headers=headers) as res: if res.status != 200: raise FileNotFoundError(f"file '{ref.swift_url}' not found") info = { From b228b5ab4a5e09f73dfcbe4183fe5b852cb3e2f4 Mon Sep 17 00:00:00 2001 From: wachsylon Date: Mon, 17 Jul 2023 17:37:14 +0200 Subject: [PATCH 2/5] Call the function at the correct position --- swiftspec/core.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/swiftspec/core.py b/swiftspec/core.py index 2e4bb8a..4c1d3fd 100644 --- a/swiftspec/core.py +++ b/swiftspec/core.py @@ -182,8 +182,9 @@ async def _ls(self, path, detail=True, **kwargs): params = { "format": "json", } - params.update(params_for_url(url)) url = f"https://{ref.host}/v1/{ref.account}" + params.update(self.params_for_url(url)) + async with session.get( url, params=params, headers=self.headers_for_url(url) ) as res: @@ -209,9 +210,9 @@ async def _ls(self, path, detail=True, **kwargs): "delimiter": "/", "prefix": prefix, } - params.update(params_for_url(url)) - url = f"https://{ref.host}/v1/{ref.account}/{ref.container}" + params.update(self.params_for_url(url)) + async with session.get( url, params=params, headers=self.headers_for_url(url) ) as res: From 1fa8dca88f21b05be1385c896d16cb4b047d702d Mon Sep 17 00:00:00 2001 From: Fabian Wachsmann Date: Thu, 20 Jul 2023 11:33:22 +0200 Subject: [PATCH 3/5] Adapt suggestions from d-70 --- swiftspec/core.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/swiftspec/core.py b/swiftspec/core.py index 4c1d3fd..8b37e75 100644 --- a/swiftspec/core.py +++ b/swiftspec/core.py @@ -142,31 +142,44 @@ def get_tokens_from_env(self): token = os.environ.get("OS_AUTH_TOKEN") url = os.environ.get("OS_STORAGE_URL") if token and url: - return [{"X-Auth-Token": token, "url": url}] + return [{"token": token, "url": url}] sig = os.environ.get("TEMP_URL_SIG") expire = os.environ.get("TEMP_URL_EXPIRES") prf = os.environ.get("TEMP_URL_PREFIX") if url and sig and expire: - return [{"url":url,"temp_url_sig": sig, "temp_url_expires": expire, "temp_url_prefix":prf}] + if prf: + return [ + { + "url": url, + "temp_url_sig": sig, + "temp_url_expires": expire, + "temp_url_prefix": prf, + } + ] + else: + return [{"url": url, "temp_url_sig": sig, "temp_url_expires": expire}] else: return [] def headers_for_url(self, url): headers = {} for auth in self.auth: - if url.startswith(auth["url"]) and "X-Auth-Token" in auth: - headers["X-Auth-Token"]=auth["X-Auth-Token"] + if url.startswith(auth["url"]) and "token" in auth: + headers["X-Auth-Token"] = auth["token"] break return headers def params_for_url(self, url): params = {} for auth in self.auth: - if url.startswith(auth["url"]) and "temp_url_sig" in auth: - params=auth.copy() - del params["url"] + if ( + url.startswith(auth["url"]) + and "temp_url_sig" in auth + and "temp_url_expires" in auth + ): + params = {k: v for k, v in auth.items if k.startswith("temp_url_")} break return params From 2ca48a8a66ce5a9dabafc294a6b256d81c724cff Mon Sep 17 00:00:00 2001 From: Fabian Wachsmann Date: Thu, 20 Jul 2023 11:42:40 +0200 Subject: [PATCH 4/5] Multiples auth for temporary urls --- swiftspec/core.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/swiftspec/core.py b/swiftspec/core.py index 8b37e75..e659ed4 100644 --- a/swiftspec/core.py +++ b/swiftspec/core.py @@ -139,10 +139,11 @@ async def set_session(self): return self._session def get_tokens_from_env(self): + auth = [] token = os.environ.get("OS_AUTH_TOKEN") url = os.environ.get("OS_STORAGE_URL") if token and url: - return [{"token": token, "url": url}] + auth.append({"token": token, "url": url}) sig = os.environ.get("TEMP_URL_SIG") expire = os.environ.get("TEMP_URL_EXPIRES") @@ -150,18 +151,19 @@ def get_tokens_from_env(self): if url and sig and expire: if prf: - return [ + auth.append( { "url": url, "temp_url_sig": sig, "temp_url_expires": expire, "temp_url_prefix": prf, } - ] + ) else: - return [{"url": url, "temp_url_sig": sig, "temp_url_expires": expire}] - else: - return [] + auth.append( + {"url": url, "temp_url_sig": sig, "temp_url_expires": expire} + ) + return auth def headers_for_url(self, url): headers = {} From c3a6b257f94eaed31baae83aa5af473531326480 Mon Sep 17 00:00:00 2001 From: Fabian Wachsmann Date: Mon, 24 Jul 2023 18:24:33 +0200 Subject: [PATCH 5/5] Only use temporary url if no token available --- swiftspec/core.py | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/swiftspec/core.py b/swiftspec/core.py index e659ed4..133c61d 100644 --- a/swiftspec/core.py +++ b/swiftspec/core.py @@ -198,11 +198,12 @@ async def _ls(self, path, detail=True, **kwargs): "format": "json", } url = f"https://{ref.host}/v1/{ref.account}" - params.update(self.params_for_url(url)) + user_params = self.params_for_url(url) + user_headers = self.headers_for_url(url) + if not (user_params and user_headers): + params.update(user_params) - async with session.get( - url, params=params, headers=self.headers_for_url(url) - ) as res: + async with session.get(url, params=params, headers=user_headers) as res: res.raise_for_status() resdata = await res.json() info = [ @@ -226,11 +227,12 @@ async def _ls(self, path, detail=True, **kwargs): "prefix": prefix, } url = f"https://{ref.host}/v1/{ref.account}/{ref.container}" - params.update(self.params_for_url(url)) + user_params = self.params_for_url(url) + user_headers = self.headers_for_url(url) + if not (user_params and user_headers): + params.update(user_params) - async with session.get( - url, params=params, headers=self.headers_for_url(url) - ) as res: + async with session.get(url, params=params, headers=user_headers) as res: res.raise_for_status() resdata = await res.json() info = [ @@ -258,6 +260,8 @@ async def _cat_file(self, path, start=None, end=None, **kwargs): ref = SWIFTRef(path) headers = self.headers_for_url(ref.http_url) params = self.params_for_url(ref.http_url) + if params and headers: + params = None if start is not None: assert start >= 0 if end is not None: @@ -288,6 +292,9 @@ async def _pipe_file(self, path, data, chunksize=50 * 2**20, **kwargs): url = ref.http_url headers = self.headers_for_url(url) params = self.params_for_url(url) + if params and headers: + params = None + headers["Content-Length"] = str(size) if self.verify_uploads: # in swift, ETag is alwas the MD5sum and will be used by the server to verify the upload @@ -363,6 +370,8 @@ async def _info(self, path, **kwargs): } headers = self.headers_for_url(ref.http_url) params = self.params_for_url(ref.http_url) + if params and headers: + params = None session = await self.set_session() async with session.head(ref.http_url, params=params, headers=headers) as res: if res.status != 200: