Merge pull request #33516 from openedx/quince-django42-upgrade

Quince django42 upgrade

.github/workflows/migrations-check.yml

@@ -18,9 +18,9 @@ jobs:
         # 'pinned' is used to install the latest patch version of Django
         # within the global constraint i.e. Django==3.2.21 in current case
         # because we have global constraint of Django<4.2 
-        django-version: ["pinned", "4.2"]
+        django-version: ["pinned"]
         mongo-version: ["4"]
-        mysql-version: ["5.7", "8"]
+        mysql-version: ["8"]
         # excluding mysql5.7 with Django 4.2 since Django 4.2 has 
         # dropped support for MySQL<8
         exclude:

Makefile

@@ -122,6 +122,8 @@ compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *
 	@# time someone tries to use the outputs.
 	sed '/^django-simple-history==/d' requirements/common_constraints.txt > requirements/common_constraints.tmp
 	mv requirements/common_constraints.tmp requirements/common_constraints.txt
+	sed 's/Django<4.0//g' requirements/common_constraints.txt > requirements/common_constraints.tmp
+	mv requirements/common_constraints.tmp requirements/common_constraints.txt
 	pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in
 	pip install -r requirements/pip.txt
 

cms/envs/common.py

@@ -831,7 +831,6 @@
 CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = ''
 CROSS_DOMAIN_CSRF_COOKIE_NAME = ''
 CSRF_TRUSTED_ORIGINS = []
-CSRF_TRUSTED_ORIGINS_WITH_SCHEME = []
 
 #################### CAPA External Code Evaluation #############################
 XQUEUE_WAITTIME_BETWEEN_REQUESTS = 5  # seconds
@@ -1086,7 +1085,8 @@
 }
 
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
-DEFAULT_HASHING_ALGORITHM = 'sha1'
+# This will be overridden through CMS config
+DEFAULT_HASHING_ALGORITHM = 'sha256'
 
 #################### Python sandbox ############################################
 

cms/envs/production.py

@@ -14,7 +14,6 @@
 import yaml
 
 from corsheaders.defaults import default_headers as corsheaders_default_headers
-import django
 from django.core.exceptions import ImproperlyConfigured
 from django.urls import reverse_lazy
 from edx_django_utils.plugins import add_plugins
@@ -237,11 +236,6 @@ def get_env_setting(setting):
 # by end users.
 CSRF_COOKIE_SECURE = ENV_TOKENS.get('CSRF_COOKIE_SECURE', False)
 
-# values are already updated above with default CSRF_TRUSTED_ORIGINS values but in
-# case of new django version these values will override.
-if django.VERSION[0] >= 4:  # for greater than django 3.2 use schemes.
-    CSRF_TRUSTED_ORIGINS = ENV_TOKENS.get('CSRF_TRUSTED_ORIGINS_WITH_SCHEME', [])
-
 #Email overrides
 MKTG_URL_LINK_MAP.update(ENV_TOKENS.get('MKTG_URL_LINK_MAP', {}))
 MKTG_URL_OVERRIDES.update(ENV_TOKENS.get('MKTG_URL_OVERRIDES', MKTG_URL_OVERRIDES))

lms/envs/common.py

@@ -1734,7 +1734,8 @@ def _make_mako_template_dirs(settings):
 
 
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
-DEFAULT_HASHING_ALGORITHM = 'sha1'
+# This will be overridden through LMS config
+DEFAULT_HASHING_ALGORITHM = 'sha256'
 
 #################### Python sandbox ############################################
 
@@ -3314,7 +3315,6 @@ def _make_locale_paths(settings):  # pylint: disable=missing-function-docstring
 # end users
 CSRF_COOKIE_SECURE = False
 CSRF_TRUSTED_ORIGINS = []
-CSRF_TRUSTED_ORIGINS_WITH_SCHEME = []
 CROSS_DOMAIN_CSRF_COOKIE_DOMAIN = ''
 CROSS_DOMAIN_CSRF_COOKIE_NAME = ''
 

lms/envs/production.py

@@ -23,7 +23,6 @@
 
 import yaml
 from corsheaders.defaults import default_headers as corsheaders_default_headers
-import django
 from django.core.exceptions import ImproperlyConfigured
 from edx_django_utils.plugins import add_plugins
 from path import Path as path
@@ -367,11 +366,6 @@ def get_env_setting(setting):
 
 # Determines which origins are trusted for unsafe requests eg. POST requests.
 CSRF_TRUSTED_ORIGINS = ENV_TOKENS.get('CSRF_TRUSTED_ORIGINS', [])
-# values are already updated above with default CSRF_TRUSTED_ORIGINS values but in
-# case of new django version these values will override.
-if django.VERSION[0] >= 4:  # for greater than django 3.2 use schemes.
-    CSRF_TRUSTED_ORIGINS = ENV_TOKENS.get('CSRF_TRUSTED_ORIGINS_WITH_SCHEME', [])
-
 ############# CORS headers for cross-domain requests #################
 
 if FEATURES.get('ENABLE_CORS_HEADERS') or FEATURES.get('ENABLE_CROSS_DOMAIN_CSRF_COOKIE'):

openedx/core/djangoapps/safe_sessions/tests/test_safe_cookie_data.py

@@ -236,7 +236,7 @@ def test_pinned_values(self):
             "|HvGnjXf1b3jU"
             "|ImExZWZiNzVlZGFmM2FkZWZmYjM4YjI0ZmZkOWU4MzExODU0MTk4NmVlNGRiYzBlODdhYWUzOGM5MzVlNzk4NjUi"
             ":1m6Hve"
-            ":OMhY2FL2pudJjSSXChtI-zR8QVA"
+            ":Pra4iochviPvKUoIV33gdVZFDgG-cMDlIYfl8iFIMaY"
         )
 
     @pytest.mark.skipif(django.VERSION[0] < 4, reason="For django42 default algorithm is sha256. No need for django32.")

openedx/core/djangoapps/theming/storage.py

@@ -192,51 +192,56 @@ def converter(matchobj):
             This requires figuring out which files the matched URL resolves
             to and calling the url() method of the storage.
             """
-            matched, url = matchobj.groups()
+            matches = matchobj.groupdict()
+            matched = matches["matched"]
+            url = matches["url"]
 
             # Ignore absolute/protocol-relative and data-uri URLs.
-            if re.match(r'^[a-z]+:', url):
+            if re.match(r"^[a-z]+:", url):
                 return matched
 
             # Ignore absolute URLs that don't point to a static file (dynamic
             # CSS / JS?). Note that STATIC_URL cannot be empty.
-            if url.startswith('/') and not url.startswith(settings.STATIC_URL):
+            if url.startswith("/") and not url.startswith(settings.STATIC_URL):
                 return matched
 
             # Strip off the fragment so a path-like fragment won't interfere.
             url_path, fragment = urldefrag(url)
 
-            if url_path.startswith('/'):
+            # Ignore URLs without a path
+            if not url_path:
+                return matched
+
+            if url_path.startswith("/"):
                 # Otherwise the condition above would have returned prematurely.
                 assert url_path.startswith(settings.STATIC_URL)
                 target_name = url_path[len(settings.STATIC_URL):]
             else:
                 # We're using the posixpath module to mix paths and URLs conveniently.
-                source_name = name if os.sep == '/' else name.replace(os.sep, '/')
+                source_name = name if os.sep == "/" else name.replace(os.sep, "/")
                 target_name = posixpath.join(posixpath.dirname(source_name), url_path)
 
             # Determine the hashed name of the target file with the storage backend.
             hashed_url = self._url(
-                self._stored_name, unquote(target_name),
-                force=True, hashed_files=hashed_files,
+                self._stored_name,
+                unquote(target_name),
+                force=True,
+                hashed_files=hashed_files,
             )
 
-            # NOTE:
-            # The line below was commented out so that absolute urls are used instead of relative urls to make themed
-            # assets work correctly.
-            #
-            # The line is commented and not removed to make future django upgrade easier and show exactly what is
-            # changed in this method override
-            #
-            #transformed_url = '/'.join(url_path.split('/')[:-1] + hashed_url.split('/')[-1:])
+            # transformed_url = "/".join(
+            #  url_path.split("/")[:-1] + hashed_url.split("/")[-1:]
+            # )
+
             transformed_url = hashed_url  # This line was added.
 
             # Restore the fragment that was stripped off earlier.
             if fragment:
-                transformed_url += ('?#' if '?#' in url else '#') + fragment
+                transformed_url += ("?#" if "?#" in url else "#") + fragment
 
             # Return the hashed version to the file
-            return template % unquote(transformed_url)
+            matches["url"] = unquote(transformed_url)
+            return template % matches
 
         return converter
 

requirements/common_constraints.txt

@@ -17,7 +17,7 @@
 
 
 # using LTS django version
-Django<4.0
+
 
 # elasticsearch>=7.14.0 includes breaking changes in it which caused issues in discovery upgrade process.
 # elastic search changelog: https://www.elastic.co/guide/en/enterprise-search/master/release-notes-7.14.0.html

requirements/constraints.txt

@@ -117,6 +117,9 @@ djangorestframework-stubs==3.14.0  # Pinned to match django-stubs. Remove this w
 # https://github.com/openedx/edx-platform/issues/31616
 libsass==0.10.0
 
+# incremental upgrade
+django-simple-history==3.4.0
+
 # greater version breaking upgrade builds
 click==8.1.6
 

requirements/edx/base.txt

@@ -60,6 +60,7 @@ backoff==1.10.0
 backports-zoneinfo[tzdata]==0.2.1
     # via
     #   celery
+    #   django
     #   icalendar
     #   kombu
 beautifulsoup4==4.12.2
@@ -79,13 +80,13 @@ boto==2.39.0
     # via
     #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/kernel.in
-boto3==1.28.62
+boto3==1.28.64
     # via
     #   -r requirements/edx/kernel.in
     #   django-ses
     #   fs-s3fs
     #   ora2
-botocore==1.31.62
+botocore==1.31.64
     # via
     #   -r requirements/edx/kernel.in
     #   boto3
@@ -169,7 +170,7 @@ cryptography==38.0.4
     #   pyopenssl
     #   snowflake-connector-python
     #   social-auth-core
-cssutils==2.7.1
+cssutils==2.9.0
     # via pynliner
 defusedxml==0.7.1
     # via
@@ -180,9 +181,8 @@ defusedxml==0.7.1
     #   social-auth-core
 deprecated==1.2.14
     # via jwcrypto
-django==3.2.22
+django==4.2.6
     # via
-    #   -c requirements/edx/../common_constraints.txt
     #   -r requirements/edx/kernel.in
     #   django-appconf
     #   django-celery-results
@@ -265,7 +265,7 @@ django-config-models==2.5.1
     #   edx-enterprise
     #   edx-name-affirmation
     #   lti-consumer-xblock
-django-cors-headers==4.2.0
+django-cors-headers==4.3.0
     # via -r requirements/edx/kernel.in
 django-countries==7.5.1
     # via
@@ -344,6 +344,7 @@ django-ses==3.5.0
     # via -r requirements/edx/bundled.in
 django-simple-history==3.4.0
     # via
+    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/kernel.in
     #   edx-enterprise
     #   edx-name-affirmation
@@ -452,7 +453,7 @@ edx-django-release-util==1.3.0
     #   -r requirements/edx/kernel.in
     #   edxval
     #   openedx-blockstore
-edx-django-sites-extensions==4.0.1
+edx-django-sites-extensions==4.0.2
     # via -r requirements/edx/kernel.in
 edx-django-utils==5.7.0
     # via
@@ -470,7 +471,7 @@ edx-django-utils==5.7.0
     #   openedx-blockstore
     #   ora2
     #   super-csv
-edx-drf-extensions==8.10.0
+edx-drf-extensions==8.12.0
     # via
     #   -r requirements/edx/kernel.in
     #   edx-completion
@@ -520,7 +521,7 @@ edx-proctoring==4.16.1
     #   edx-proctoring-proctortrack
 edx-rbac==1.8.0
     # via edx-enterprise
-edx-rest-api-client==5.6.0
+edx-rest-api-client==5.6.1
     # via
     #   -r requirements/edx/kernel.in
     #   edx-enterprise
@@ -793,7 +794,7 @@ openedx-mongodbproxy==0.2.0
     # via -r requirements/edx/kernel.in
 optimizely-sdk==4.1.1
     # via -r requirements/edx/bundled.in
-ora2==5.5.4
+ora2==5.5.5
     # via -r requirements/edx/bundled.in
 oscrypto==1.3.0
     # via snowflake-connector-python
@@ -841,7 +842,7 @@ polib==1.2.0
     # via edx-i18n-tools
 prompt-toolkit==3.0.39
     # via click-repl
-psutil==5.9.5
+psutil==5.9.6
     # via
     #   -r requirements/edx/paver.txt
     #   edx-django-utils
@@ -946,7 +947,6 @@ pytz==2022.7.1
     #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/kernel.in
     #   babel
-    #   django
     #   django-ses
     #   djangorestframework
     #   drf-yasg
@@ -1015,7 +1015,7 @@ requests-oauthlib==1.3.1
     # via
     #   -r requirements/edx/kernel.in
     #   social-auth-core
-rpds-py==0.10.4
+rpds-py==0.10.6
     # via
     #   jsonschema
     #   referencing
@@ -1040,7 +1040,7 @@ scipy==1.7.3
     #   openedx-calc
 semantic-version==2.10.0
     # via edx-drf-extensions
-shapely==2.0.1
+shapely==2.0.2
     # via -r requirements/edx/kernel.in
 simplejson==3.19.2
     # via
@@ -1084,7 +1084,7 @@ slumber==0.7.1
     #   edx-bulk-grades
     #   edx-enterprise
     #   edx-rest-api-client
-snowflake-connector-python==3.2.1
+snowflake-connector-python==3.3.0
     # via edx-enterprise
 social-auth-app-django==5.0.0
     # via
@@ -1221,7 +1221,7 @@ xblock[django]==1.8.1
     #   xblock-google-drive
     #   xblock-poll
     #   xblock-utils
-xblock-drag-and-drop-v2==3.2.0
+xblock-drag-and-drop-v2==3.2.1
     # via -r requirements/edx/bundled.in
 xblock-google-drive==0.4.0
     # via -r requirements/edx/bundled.in

requirements/edx/coverage.txt

@@ -8,7 +8,7 @@ chardet==5.2.0
     # via diff-cover
 coverage==7.3.2
     # via -r requirements/edx/coverage.in
-diff-cover==7.7.0
+diff-cover==8.0.0
     # via -r requirements/edx/coverage.in
 jinja2==3.1.2
     # via diff-cover