From 65e73299f01bd2b97999acddd64c4056fe37aaa8 Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Wed, 3 May 2017 06:04:06 +0200 Subject: [PATCH 01/11] Ignore egg-info. --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a63aee44..575a79bc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ -_trial_temp/ +*.egg-info/ .testrepository .venv-* +_trial_temp/ From edc3abce241aee8073b6ccd92c66a310a7087c73 Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Wed, 3 May 2017 06:05:13 +0200 Subject: [PATCH 02/11] Add an integration test for objects with encoding-requiring names. --- txaws/testing/s3_tests.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/txaws/testing/s3_tests.py b/txaws/testing/s3_tests.py index dce7eb9d..06ed46e3 100644 --- a/txaws/testing/s3_tests.py +++ b/txaws/testing/s3_tests.py @@ -176,4 +176,29 @@ def test_put_object_body_producer(self): self.assertEqual(object_data, retrieved) + @inlineCallbacks + def test_object_encoded_chars(self): + """ + C{get_object} and C{put_object} suceed with an object name that + requires encoding. + """ + bucket_name = str(uuid4()) + object_names = [ + b'object:with:colons', + b'object with spaces', + u'\N{SNOWMAN}'.encode('utf-8'), + ] + object_data = b'some text' + object_type = b'application/x-txaws-integration-testing' + + client = get_client(self) + yield client.create_bucket(bucket_name) + for object_name in object_names: + yield client.put_object( + bucket_name, object_name, object_data, + content_type=object_type) + retrieved = yield client.get_object(bucket_name, object_name) + self.assertEqual(object_data, retrieved) + + return S3IntegrationTests From fe262e05016f0a96aca031b8dd1bcb66bcc85c52 Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Wed, 3 May 2017 06:06:21 +0200 Subject: [PATCH 03/11] Add an new method to the un-url-encoded path, and use it for auth instead. --- txaws/client/base.py | 10 +++++++++- txaws/s3/client.py | 4 ++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/txaws/client/base.py b/txaws/client/base.py index cf598127..9b940f0e 100644 --- a/txaws/client/base.py +++ b/txaws/client/base.py @@ -270,6 +270,14 @@ def get_encoded_path(self): ) + def get_joined_path(self): + """ + @return: The path component, un-urlencoded, but joined by slashes. + @rtype: L{bytes} + """ + return b'/' + b'/'.join(seg.encode('utf-8') for seg in self.path) + + def get_encoded_query(self): """ @return: The encoded query component. @@ -412,7 +420,7 @@ def _canonical_request(self, headers): return _auth_v4._CanonicalRequest.from_request_components( method=self._details.method, url=( - self._details.url_context.get_encoded_path() + + self._details.url_context.get_joined_path() + b"?" + self._details.url_context.get_encoded_query() ), diff --git a/txaws/s3/client.py b/txaws/s3/client.py index c2041717..47ec49c8 100644 --- a/txaws/s3/client.py +++ b/txaws/s3/client.py @@ -772,7 +772,7 @@ def sign(self, headers, data, url_context, instant, method, if data is None: request = _auth_v4._CanonicalRequest.from_request_components( method=method, - url=url_context.get_encoded_path(), + url=url_context.get_joined_path(), headers=headers, headers_to_sign=('host', 'x-amz-date'), payload_hash=None, @@ -780,7 +780,7 @@ def sign(self, headers, data, url_context, instant, method, else: request = _auth_v4._CanonicalRequest.from_request_components_and_payload( method=method, - url=url_context.get_encoded_path(), + url=url_context.get_joined_path(), headers=headers, headers_to_sign=('host', 'x-amz-date'), payload=data, From a9b3c0b8ae64bc24b1d9cdf1fea8486357c10685 Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Wed, 3 May 2017 06:06:41 +0200 Subject: [PATCH 04/11] Clerical tweak. --- txaws/_auth_v4.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/txaws/_auth_v4.py b/txaws/_auth_v4.py index a74f1472..3ba94506 100644 --- a/txaws/_auth_v4.py +++ b/txaws/_auth_v4.py @@ -284,7 +284,7 @@ def serialize(self): request. @rtype: L{str} """ - return '\n'.join(attr.astuple(self)) + return b'\n'.join(attr.astuple(self)) def hash(self): """ From ccc5aae268c1c84c3e69e6a16ca3cf8ac8c18053 Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Fri, 5 May 2017 08:16:33 +0200 Subject: [PATCH 05/11] Fix typo in docstring. --- txaws/testing/s3_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/txaws/testing/s3_tests.py b/txaws/testing/s3_tests.py index 06ed46e3..cb660e1f 100644 --- a/txaws/testing/s3_tests.py +++ b/txaws/testing/s3_tests.py @@ -179,7 +179,7 @@ def test_put_object_body_producer(self): @inlineCallbacks def test_object_encoded_chars(self): """ - C{get_object} and C{put_object} suceed with an object name that + C{get_object} and C{put_object} succeed with an object name that requires encoding. """ bucket_name = str(uuid4()) From d212451ae5560315481ca24ecee4a6900577a1f7 Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Fri, 5 May 2017 08:16:40 +0200 Subject: [PATCH 06/11] Fix MemoryS3 methods returning synchronous results. --- txaws/testing/s3.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/txaws/testing/s3.py b/txaws/testing/s3.py index 6b9b05b7..3279745e 100644 --- a/txaws/testing/s3.py +++ b/txaws/testing/s3.py @@ -96,11 +96,11 @@ def get_bucket(self, bucket): pieces = self._state.buckets[bucket] except KeyError: return fail(S3Error("", 400)) - return pieces["listing"] + return succeed(pieces["listing"]) @_rate_limited def get_bucket_location(self, bucket): - return b"" + return succeed(b"") @_rate_limited def put_object( @@ -144,11 +144,12 @@ def put_object( def _store_object(self, bucket, obj, data): self._state.objects[bucket, obj] = data + return succeed(None) @_rate_limited def get_object(self, bucket, object_name): - return self._state.objects[bucket, object_name] + return succeed(self._state.objects[bucket, object_name]) @_rate_limited def delete_object(self, bucket, object_name): @@ -158,6 +159,7 @@ def delete_object(self, bucket, object_name): if item.key == object_name: contents.remove(item) break + return succeed(None) @attr.s From 72c465b4bb0aaf15a46fb371781a663510f24eff Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Fri, 5 May 2017 08:17:05 +0200 Subject: [PATCH 07/11] Avoid asserting in a loop. --- txaws/testing/s3_tests.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/txaws/testing/s3_tests.py b/txaws/testing/s3_tests.py index cb660e1f..ee0e8bb0 100644 --- a/txaws/testing/s3_tests.py +++ b/txaws/testing/s3_tests.py @@ -197,8 +197,11 @@ def test_object_encoded_chars(self): yield client.put_object( bucket_name, object_name, object_data, content_type=object_type) - retrieved = yield client.get_object(bucket_name, object_name) - self.assertEqual(object_data, retrieved) + retrieved = yield gatherResults( + [client.get_object(bucket_name, object_name) + for object_name in object_names], + consumeErrors=True) + self.assertEqual([object_data] * len(object_names), retrieved) return S3IntegrationTests From 2f24d85286b0c71b1807a5210a2a1096550a1b29 Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Fri, 5 May 2017 08:20:21 +0200 Subject: [PATCH 08/11] Make this private. --- txaws/client/base.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/txaws/client/base.py b/txaws/client/base.py index 9b940f0e..2538adf8 100644 --- a/txaws/client/base.py +++ b/txaws/client/base.py @@ -270,14 +270,6 @@ def get_encoded_path(self): ) - def get_joined_path(self): - """ - @return: The path component, un-urlencoded, but joined by slashes. - @rtype: L{bytes} - """ - return b'/' + b'/'.join(seg.encode('utf-8') for seg in self.path) - - def get_encoded_query(self): """ @return: The encoded query component. @@ -306,6 +298,17 @@ def get_encoded_url(self): return b"%(scheme)s://%(host)s:%(port)d%(path)s%(query)s" % params +def _get_joined_path(ctx): + """ + @type ctx: L{_URLContext} + @param ctx: A URL context. + + @return: The path component, un-urlencoded, but joined by slashes. + @rtype: L{bytes} + """ + return b'/' + b'/'.join(seg.encode('utf-8') for seg in ctx.path) + + @attr.s class RequestDetails(object): """ @@ -420,7 +423,7 @@ def _canonical_request(self, headers): return _auth_v4._CanonicalRequest.from_request_components( method=self._details.method, url=( - self._details.url_context.get_joined_path() + + _get_joined_path(self._details.url_context) + b"?" + self._details.url_context.get_encoded_query() ), From b83772ad75e64ba4429e3d1218635ceb7e61dc1d Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Fri, 5 May 2017 08:25:35 +0200 Subject: [PATCH 09/11] Revert changes to deprecated code. --- txaws/s3/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/txaws/s3/client.py b/txaws/s3/client.py index 47ec49c8..c2041717 100644 --- a/txaws/s3/client.py +++ b/txaws/s3/client.py @@ -772,7 +772,7 @@ def sign(self, headers, data, url_context, instant, method, if data is None: request = _auth_v4._CanonicalRequest.from_request_components( method=method, - url=url_context.get_joined_path(), + url=url_context.get_encoded_path(), headers=headers, headers_to_sign=('host', 'x-amz-date'), payload_hash=None, @@ -780,7 +780,7 @@ def sign(self, headers, data, url_context, instant, method, else: request = _auth_v4._CanonicalRequest.from_request_components_and_payload( method=method, - url=url_context.get_joined_path(), + url=url_context.get_encoded_path(), headers=headers, headers_to_sign=('host', 'x-amz-date'), payload=data, From 644775d41fd05fd24f3f50a743287e43de2009ad Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Sat, 13 May 2017 11:28:37 +0200 Subject: [PATCH 10/11] Add a news fragment. --- txaws/newsfragments/20.bugfix | 1 + 1 file changed, 1 insertion(+) create mode 100644 txaws/newsfragments/20.bugfix diff --git a/txaws/newsfragments/20.bugfix b/txaws/newsfragments/20.bugfix new file mode 100644 index 00000000..3cc2ffbe --- /dev/null +++ b/txaws/newsfragments/20.bugfix @@ -0,0 +1 @@ +txaws now correctly signs requests with paths that require urlencoding. From b76c6119db47393d4431b7ab48306da82c73a35e Mon Sep 17 00:00:00 2001 From: Tristan Seligmann Date: Sat, 13 May 2017 11:31:29 +0200 Subject: [PATCH 11/11] Link to issue about improving from_request_components API. --- txaws/client/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/txaws/client/base.py b/txaws/client/base.py index 2538adf8..f90458c1 100644 --- a/txaws/client/base.py +++ b/txaws/client/base.py @@ -423,6 +423,8 @@ def _canonical_request(self, headers): return _auth_v4._CanonicalRequest.from_request_components( method=self._details.method, url=( + # We need to pass an unfortunate version of the path here: see + # https://github.com/twisted/txaws/issues/70 _get_joined_path(self._details.url_context) + b"?" + self._details.url_context.get_encoded_query()