From 421ca7cca559838656c03daf6545295767469027 Mon Sep 17 00:00:00 2001 From: Dirk Doesburg Date: Wed, 13 Nov 2024 19:29:33 +0100 Subject: [PATCH] Update PublicS3Storage for django-storages 1.14.4 This changes the cloudfront cache policies to allow passing through the response-content-disposition query parameter, so we can use that to make a URL without cloudfront signature that will be downloaded. --- infra/terraform/concrexit/modules/cdn/main.tf | 33 +++++++- poetry.lock | 78 ++++++++++--------- pyproject.toml | 2 +- website/thaliawebsite/storage/backend.py | 20 ++--- 4 files changed, 82 insertions(+), 51 deletions(-) diff --git a/infra/terraform/concrexit/modules/cdn/main.tf b/infra/terraform/concrexit/modules/cdn/main.tf index 47dc14089..0d112141c 100644 --- a/infra/terraform/concrexit/modules/cdn/main.tf +++ b/infra/terraform/concrexit/modules/cdn/main.tf @@ -35,6 +35,36 @@ resource "aws_cloudfront_response_headers_policy" "static_cors_headers" { } } +resource "aws_cloudfront_cache_policy" "public_storage" { + name = "${var.customer}-${var.stage}-concrexit-public-storage" + comment = "Cache policy that forwards the response-content-disposition header." + min_ttl = 1 + default_ttl = 86400 + max_ttl = 31536000 + + parameters_in_cache_key_and_forwarded_to_origin { + cookies_config { + cookie_behavior = "none" + } + + headers_config { + header_behavior = "none" + } + + query_strings_config { + # This is required to allow using the response-content-disposition query parameter + # to cause a public file (without a signed URL) to be received as an attachment. + query_string_behavior = "whitelist" + query_strings { + items = ["response-content-disposition"] + } + } + + enable_accept_encoding_gzip = true + enable_accept_encoding_brotli = true + } +} + resource "aws_cloudfront_distribution" "this" { aliases = [var.webdomain] @@ -59,7 +89,6 @@ resource "aws_cloudfront_distribution" "this" { trusted_key_groups = [aws_cloudfront_key_group.this.id] cache_policy_id = data.aws_cloudfront_cache_policy.caching_optimized.id - } ordered_cache_behavior { @@ -70,7 +99,7 @@ resource "aws_cloudfront_distribution" "this" { cached_methods = ["GET", "HEAD"] compress = true - cache_policy_id = data.aws_cloudfront_cache_policy.caching_optimized.id + cache_policy_id = aws_cloudfront_cache_policy.public_storage.id } ordered_cache_behavior { diff --git a/poetry.lock b/poetry.lock index ea2bdd158..c7eb13417 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,13 +2,13 @@ [[package]] name = "amqp" -version = "5.2.0" +version = "5.3.1" description = "Low-level AMQP client for Python (fork of amqplib)." optional = false python-versions = ">=3.6" files = [ - {file = "amqp-5.2.0-py3-none-any.whl", hash = "sha256:827cb12fb0baa892aad844fd95258143bce4027fdac4fccddbc43330fd281637"}, - {file = "amqp-5.2.0.tar.gz", hash = "sha256:a1ecff425ad063ad42a486c902807d1482311481c8ad95a72694b2975e75f7fd"}, + {file = "amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2"}, + {file = "amqp-5.3.1.tar.gz", hash = "sha256:cddc00c725449522023bad949f70fff7b48f0b1ade74d170a6f10ab044739432"}, ] [package.dependencies] @@ -97,13 +97,13 @@ files = [ [[package]] name = "async-timeout" -version = "4.0.3" +version = "5.0.1" description = "Timeout context manager for asyncio programs" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, - {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, + {file = "async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c"}, + {file = "async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3"}, ] [[package]] @@ -275,13 +275,13 @@ crt = ["awscrt (==0.21.2)"] [[package]] name = "cachecontrol" -version = "0.14.0" +version = "0.14.1" description = "httplib2 caching for requests" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "cachecontrol-0.14.0-py3-none-any.whl", hash = "sha256:f5bf3f0620c38db2e5122c0726bdebb0d16869de966ea6a2befe92470b740ea0"}, - {file = "cachecontrol-0.14.0.tar.gz", hash = "sha256:7db1195b41c81f8274a7bbd97c956f44e8348265a1bc7641c37dfebc39f0c938"}, + {file = "cachecontrol-0.14.1-py3-none-any.whl", hash = "sha256:65e3abd62b06382ce3894df60dde9e0deb92aeb734724f68fa4f3b91e97206b9"}, + {file = "cachecontrol-0.14.1.tar.gz", hash = "sha256:06ef916a1e4eb7dba9948cdfc9c76e749db2e02104a9a1277e8b642591a0f717"}, ] [package.dependencies] @@ -289,7 +289,7 @@ msgpack = ">=0.5.2,<2.0.0" requests = ">=2.16.0" [package.extras] -dev = ["CacheControl[filecache,redis]", "black", "build", "cherrypy", "furo", "mypy", "pytest", "pytest-cov", "sphinx", "sphinx-copybutton", "tox", "types-redis", "types-requests"] +dev = ["CacheControl[filecache,redis]", "build", "cherrypy", "codespell[tomli]", "furo", "mypy", "pytest", "pytest-cov", "ruff", "sphinx", "sphinx-copybutton", "tox", "types-redis", "types-requests"] filecache = ["filelock (>=3.8.0)"] redis = ["redis (>=2.10.5)"] @@ -1181,13 +1181,13 @@ docs = ["sphinx", "sphinx-rtd-theme"] [[package]] name = "django-storages" -version = "1.14.3" +version = "1.14.4" description = "Support for many storage backends in Django" optional = false python-versions = ">=3.7" files = [ - {file = "django-storages-1.14.3.tar.gz", hash = "sha256:95a12836cd998d4c7a4512347322331c662d9114c4344f932f5e9c0fce000608"}, - {file = "django_storages-1.14.3-py3-none-any.whl", hash = "sha256:31f263389e95ce3a1b902fb5f739a7ed32895f7d8b80179fe7453ecc0dfe102e"}, + {file = "django-storages-1.14.4.tar.gz", hash = "sha256:69aca94d26e6714d14ad63f33d13619e697508ee33ede184e462ed766dc2a73f"}, + {file = "django_storages-1.14.4-py3-none-any.whl", hash = "sha256:d61930acb4a25e3aebebc6addaf946a3b1df31c803a6bf1af2f31c9047febaa3"}, ] [package.dependencies] @@ -1377,13 +1377,13 @@ python-dateutil = ">=2.7" [[package]] name = "google-api-core" -version = "2.22.0" +version = "2.23.0" description = "Google API client core library" optional = false python-versions = ">=3.7" files = [ - {file = "google_api_core-2.22.0-py3-none-any.whl", hash = "sha256:a6652b6bd51303902494998626653671703c420f6f4c88cfd3f50ed723e9d021"}, - {file = "google_api_core-2.22.0.tar.gz", hash = "sha256:26f8d76b96477db42b55fd02a33aae4a42ec8b86b98b94969b7333a2c828bf35"}, + {file = "google_api_core-2.23.0-py3-none-any.whl", hash = "sha256:c20100d4c4c41070cf365f1d8ddf5365915291b5eb11b83829fbd1c999b5122f"}, + {file = "google_api_core-2.23.0.tar.gz", hash = "sha256:2ceb087315e6af43f256704b871d99326b1f12a9d6ce99beaedec99ba26a0ace"}, ] [package.dependencies] @@ -1424,13 +1424,13 @@ uritemplate = ">=3.0.1,<5" [[package]] name = "google-auth" -version = "2.35.0" +version = "2.36.0" description = "Google Authentication Library" optional = false python-versions = ">=3.7" files = [ - {file = "google_auth-2.35.0-py2.py3-none-any.whl", hash = "sha256:25df55f327ef021de8be50bad0dfd4a916ad0de96da86cd05661c9297723ad3f"}, - {file = "google_auth-2.35.0.tar.gz", hash = "sha256:f4c64ed4e01e8e8b646ef34c018f8bf3338df0c8e37d8b3bba40e7f574a3278a"}, + {file = "google_auth-2.36.0-py2.py3-none-any.whl", hash = "sha256:51a15d47028b66fd36e5c64a82d2d57480075bccc7da37cde257fc94177a61fb"}, + {file = "google_auth-2.36.0.tar.gz", hash = "sha256:545e9618f2df0bcbb7dcbc45a546485b1212624716975a1ea5ae8149ce769ab1"}, ] [package.dependencies] @@ -1578,13 +1578,13 @@ requests = ["requests (>=2.18.0,<3.0.0dev)"] [[package]] name = "googleapis-common-protos" -version = "1.65.0" +version = "1.66.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis_common_protos-1.65.0-py2.py3-none-any.whl", hash = "sha256:2972e6c496f435b92590fd54045060867f3fe9be2c82ab148fc8885035479a63"}, - {file = "googleapis_common_protos-1.65.0.tar.gz", hash = "sha256:334a29d07cddc3aa01dee4988f9afd9b2916ee2ff49d6b757155dc0d197852c0"}, + {file = "googleapis_common_protos-1.66.0-py2.py3-none-any.whl", hash = "sha256:d7abcd75fabb2e0ec9f74466401f6c119a0b498e27370e9be4c94cb7e382b8ed"}, + {file = "googleapis_common_protos-1.66.0.tar.gz", hash = "sha256:c3e7b33d15fdca5374cc0a7346dd92ffa847425cc4ea941d970f13680052ec8c"}, ] [package.dependencies] @@ -1731,13 +1731,13 @@ test = ["coverage", "hypothesis", "pytest", "pytz"] [[package]] name = "identify" -version = "2.6.1" +version = "2.6.2" description = "File identification library for Python" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "identify-2.6.1-py2.py3-none-any.whl", hash = "sha256:53863bcac7caf8d2ed85bd20312ea5dcfc22226800f6d6881f232d861db5a8f0"}, - {file = "identify-2.6.1.tar.gz", hash = "sha256:91478c5fb7c3aac5ff7bf9b4344f803843dc586832d5f110d672b19aa1984c98"}, + {file = "identify-2.6.2-py2.py3-none-any.whl", hash = "sha256:c097384259f49e372f4ea00a19719d95ae27dd5ff0fd77ad630aa891306b82f3"}, + {file = "identify-2.6.2.tar.gz", hash = "sha256:fab5c716c24d7a789775228823797296a2994b075fb6080ac83a102772a98cbd"}, ] [package.extras] @@ -1972,13 +1972,13 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "packaging" -version = "24.1" +version = "24.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, - {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, + {file = "packaging-24.2-py3-none-any.whl", hash = "sha256:09abb1bccd265c01f4a3aa3f7a7db064b36514d2cba19a2f694fe6150451a759"}, + {file = "packaging-24.2.tar.gz", hash = "sha256:c228a6dc5e932d346bc5739379109d49e8853dd8223571c7c5b55260edc0b97f"}, ] [[package]] @@ -1994,13 +1994,13 @@ files = [ [[package]] name = "phonenumbers" -version = "8.13.48" +version = "8.13.49" description = "Python version of Google's common library for parsing, formatting, storing and validating international phone numbers." optional = false python-versions = "*" files = [ - {file = "phonenumbers-8.13.48-py2.py3-none-any.whl", hash = "sha256:5c51939acefa390eb74119750afb10a85d3c628dc83fd62c52d6f532fcf5d205"}, - {file = "phonenumbers-8.13.48.tar.gz", hash = "sha256:62d8df9b0f3c3c41571c6b396f044ddd999d61631534001b8be7fdf7ba1b18f3"}, + {file = "phonenumbers-8.13.49-py2.py3-none-any.whl", hash = "sha256:e17140955ab3d8f9580727372ea64c5ada5327932d6021ef6fd203c3db8c8139"}, + {file = "phonenumbers-8.13.49.tar.gz", hash = "sha256:e608ccb61f0bd42e6db1d2c421f7c22186b88f494870bf40aa31d1a2718ab0ae"}, ] [[package]] @@ -2657,13 +2657,13 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] [[package]] name = "sentry-sdk" -version = "2.17.0" +version = "2.18.0" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = ">=3.6" files = [ - {file = "sentry_sdk-2.17.0-py2.py3-none-any.whl", hash = "sha256:625955884b862cc58748920f9e21efdfb8e0d4f98cca4ab0d3918576d5b606ad"}, - {file = "sentry_sdk-2.17.0.tar.gz", hash = "sha256:dd0a05352b78ffeacced73a94e86f38b32e2eae15fff5f30ca5abb568a72eacf"}, + {file = "sentry_sdk-2.18.0-py2.py3-none-any.whl", hash = "sha256:ee70e27d1bbe4cd52a38e1bd28a5fadb9b17bc29d91b5f2b97ae29c0a7610442"}, + {file = "sentry_sdk-2.18.0.tar.gz", hash = "sha256:0dc21febd1ab35c648391c664df96f5f79fb0d92d7d4225cd9832e53a617cafd"}, ] [package.dependencies] @@ -2691,9 +2691,11 @@ httpx = ["httpx (>=0.16.0)"] huey = ["huey (>=2)"] huggingface-hub = ["huggingface-hub (>=0.22)"] langchain = ["langchain (>=0.0.210)"] +launchdarkly = ["launchdarkly-server-sdk (>=9.8.0)"] litestar = ["litestar (>=2.0.0)"] loguru = ["loguru (>=0.5)"] openai = ["openai (>=1.0.0)", "tiktoken (>=0.3.0)"] +openfeature = ["openfeature-sdk (>=0.7.1)"] opentelemetry = ["opentelemetry-distro (>=0.35b0)"] opentelemetry-experimental = ["opentelemetry-distro"] pure-eval = ["asttokens", "executing", "pure-eval"] @@ -2896,4 +2898,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.11" -content-hash = "dd41eaab0f446f1eb1b889788984a0c0265d07a447a024ec89407efa4041e552" +content-hash = "309f40b1f79b9452e4c2cc2a768365425a56bf0e3192e9707bcb927c4d2288df" diff --git a/pyproject.toml b/pyproject.toml index 0bb75a3f8..bf8798b3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ django-debug-toolbar = "4.4.6" django-admin-autocomplete-filter = "0.7.1" django-bootstrap5 = "^22.1" django-tinymce = "4.1.0" -django-storages = "1.14.3" +django-storages = "1.14.4" django-drf-filepond = "0.5.0" django-filepond-widget = "0.9.0" django-thumbnails = "0.7.0" diff --git a/website/thaliawebsite/storage/backend.py b/website/thaliawebsite/storage/backend.py index 205de4da9..8c477f6e5 100644 --- a/website/thaliawebsite/storage/backend.py +++ b/website/thaliawebsite/storage/backend.py @@ -23,14 +23,11 @@ class PublicS3Storage(S3RenameMixin, S3Boto3Storage): PUBLIC_MEDIA_LOCATION publicly, despite the objects in the bucket having a "private" ACL. Hence, we can still use the default "private" ACL to prevent people from using the S3 bucket directly. - - To support the "ResponseContentDisposition" parameter for the `attachment` argument - to `PublicS3Storage.url` we do still need to sign the url, but in other cases we can - strip the signing parameters from urls. """ location = settings.PUBLIC_MEDIA_LOCATION file_overwrite = False + querystring_auth = False # Cloudfront will have a behavior to serve files in PUBLIC_MEDIA_LOCATION publicly, # despite the objects in the bucket having a private ACL. Hence, we can use the @@ -39,17 +36,18 @@ class PublicS3Storage(S3RenameMixin, S3Boto3Storage): def url(self, name, attachment=False, expire_seconds=None): params = {} if attachment: + # The S3 bucket will add the Content-Disposition header to the response + # if the signed URL contains the response-content-disposition query parameter. + # Cloudfront's cache policy is configured to pass this parameter through to + # the origin. Otherwise, it wouldn't end up at S3. + # The `ResponseContentDisposition` parameter as used in the private storage + # does not work if there's no signature present. params[ - "ResponseContentDisposition" + "response-content-disposition" ] = f'attachment; filename="{attachment}"' url = super().url(name, params, expire=expire_seconds) - if not attachment: - # The signature is required even for public media in order - # to support the "ResponseContentDisposition" parameter. - return self._strip_signing_parameters(url) - return url @@ -60,6 +58,8 @@ class PrivateS3Storage(S3RenameMixin, S3Boto3Storage): def url(self, name, attachment=False, expire_seconds=None): params = {} if attachment: + # Cloudfront will add the Content-Disposition header to the response + # if the signed URL contains the ResponseContentDisposition query parameter. params[ "ResponseContentDisposition" ] = f'attachment; filename="{attachment}"'