diff --git a/.annotation_safe_list.yml b/.annotation_safe_list.yml index c57eeeb2503d..e91fe39cd613 100644 --- a/.annotation_safe_list.yml +++ b/.annotation_safe_list.yml @@ -142,8 +142,6 @@ workflow.AssessmentWorkflowStep: # Via edx-celeryutils celery_utils.ChordData: ".. no_pii:": "No PII" -celery_utils.FailedTask: - ".. no_pii:": "No PII" # Via completion XBlock completion.BlockCompletion: diff --git a/.eslintignore b/.eslintignore index 9044a0cc711f..b7754944dca5 100644 --- a/.eslintignore +++ b/.eslintignore @@ -20,9 +20,21 @@ test_root/staticfiles common/static/xmodule -# Symlinks into xmodule/js +# Various intra-repo symlinks that we've added over the years to duct-tape the JS build together. +# Ignore them so that we're not double-counting these violations. +cms/static/edx-ui-toolkit cms/static/xmodule_js +lms/static/common +lms/static/course_bookmarks +lms/static/course_experience +lms/static/course_search +lms/static/discussion +lms/static/edx-ui-toolkit +lms/static/learner_profile +lms/static/support +lms/static/teams lms/static/xmodule_js +xmodule/js/common_static # Mako templates that generate .js files diff --git a/.github/workflows/js-tests.yml b/.github/workflows/js-tests.yml index c9d2d7ab1191..463352e1c552 100644 --- a/.github/workflows/js-tests.yml +++ b/.github/workflows/js-tests.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - node-version: [18, 20] + node-version: [20] python-version: - "3.11" @@ -28,7 +28,7 @@ jobs: node-version: ${{ matrix.node-version }} - name: Setup npm - run: npm i -g npm@10.5.x + run: npm i -g npm@10.7.x - name: Install Firefox 123.0 run: | @@ -64,13 +64,13 @@ jobs: make base-requirements - uses: c-hive/gha-npm-cache@v1 + + - name: Install npm + run: npm ci + - name: Run JS Tests - env: - TEST_SUITE: js-unit - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh run: | - npm install -g jest - xvfb-run --auto-servernum ./scripts/all-tests.sh + npm run test - name: Save Job Artifacts uses: actions/upload-artifact@v4 diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 310f9f83bf3d..2452f54da14b 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -60,16 +60,29 @@ jobs: PIP_SRC: ${{ runner.temp }} run: | make test-requirements - + + - name: Install npm + env: + PIP_SRC: ${{ runner.temp }} + run: npm ci + + - name: Install python packages + env: + PIP_SRC: ${{ runner.temp }} + run: | + pip install -e . + - name: Run Quality Tests env: - TEST_SUITE: quality - SCRIPT_TO_RUN: ./scripts/generic-ci-tests.sh PIP_SRC: ${{ runner.temp }} TARGET_BRANCH: ${{ github.base_ref }} run: | - ./scripts/all-tests.sh - + make pycodestyle + npm run lint + make xsslint + make pii_check + make check_keywords + - name: Save Job Artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/static-assets-check.yml b/.github/workflows/static-assets-check.yml index e08b2dce8127..502dddce08ad 100644 --- a/.github/workflows/static-assets-check.yml +++ b/.github/workflows/static-assets-check.yml @@ -15,8 +15,8 @@ jobs: os: [ubuntu-24.04] python-version: - "3.11" - node-version: [18, 20] - npm-version: [10.5.x] + node-version: [20] + npm-version: [10.7.x] mongo-version: - "7.0" diff --git a/.nvmrc b/.nvmrc index 3c032078a4a2..209e3ef4b624 100644 --- a/.nvmrc +++ b/.nvmrc @@ -1 +1 @@ -18 +20 diff --git a/.pii_annotations.yml b/.pii_annotations.yml index 328520738f10..9000115a253e 100644 --- a/.pii_annotations.yml +++ b/.pii_annotations.yml @@ -1,7 +1,7 @@ source_path: ./ report_path: pii_report safelist_path: .annotation_safe_list.yml -coverage_target: 94.5 +coverage_target: 85.3 # See OEP-30 for more information on these values and what they mean: # https://open-edx-proposals.readthedocs.io/en/latest/oep-0030-arch-pii-markup-and-auditing.html#docstring-annotations annotations: diff --git a/.stylelintignore b/.stylelintignore deleted file mode 100644 index cd53bacf3cf9..000000000000 --- a/.stylelintignore +++ /dev/null @@ -1,5 +0,0 @@ -xmodule/css -common/static/sass/bourbon -common/static/xmodule/modules/css -common/test/test-theme -lms/static/sass/vendor diff --git a/Dockerfile b/Dockerfile index 75a716177fdc..a67c46738bcc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,36 +48,36 @@ RUN echo "locales locales/locales_to_be_generated multiselect en_US.UTF-8 UTF-8" # Setting up ppa deadsnakes to get python 3.11 RUN apt-get update && \ - apt-get install -y software-properties-common && \ - apt-add-repository -y ppa:deadsnakes/ppa + apt-get install -y software-properties-common && \ + apt-add-repository -y ppa:deadsnakes/ppa # Install requirements that are absolutely necessary RUN apt-get update && \ apt-get -y dist-upgrade && \ apt-get -y install --no-install-recommends \ - python3-pip \ - python3.11 \ - # python3-dev: required for building mysqlclient python package - python3.11-dev \ - python3.11-venv \ - libpython3.11 \ - libpython3.11-stdlib \ - libmysqlclient21 \ - # libmysqlclient-dev: required for building mysqlclient python package - libmysqlclient-dev \ - pkg-config \ - libssl1.1 \ - libxmlsec1-openssl \ - # lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16 - lynx \ - ntp \ - git \ - build-essential \ - gettext \ - gfortran \ - graphviz \ - locales \ - swig \ + python3-pip \ + python3.11 \ + # python3-dev: required for building mysqlclient python package + python3.11-dev \ + python3.11-venv \ + libpython3.11 \ + libpython3.11-stdlib \ + libmysqlclient21 \ + # libmysqlclient-dev: required for building mysqlclient python package + libmysqlclient-dev \ + pkg-config \ + libssl1.1 \ + libxmlsec1-openssl \ + # lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16 + lynx \ + ntp \ + git \ + build-essential \ + gettext \ + gfortran \ + graphviz \ + locales \ + swig \ && \ apt-get clean all && \ rm -rf /var/lib/apt/* @@ -93,19 +93,19 @@ FROM minimal-system as builder-production RUN apt-get update && \ apt-get -y install --no-install-recommends \ - curl \ - libssl-dev \ - libffi-dev \ - libfreetype6-dev \ - libgeos-dev \ - libgraphviz-dev \ - libjpeg8-dev \ - liblapack-dev \ - libpng-dev \ - libsqlite3-dev \ - libxml2-dev \ - libxmlsec1-dev \ - libxslt1-dev + curl \ + libssl-dev \ + libffi-dev \ + libfreetype6-dev \ + libgeos-dev \ + libgraphviz-dev \ + libjpeg8-dev \ + liblapack-dev \ + libpng-dev \ + libsqlite3-dev \ + libxml2-dev \ + libxmlsec1-dev \ + libxslt1-dev # Setup python virtual environment # It is already 'activated' because $VIRTUAL_ENV/bin was put on $PATH @@ -118,8 +118,8 @@ RUN pip install -r requirements/pip.txt RUN pip install -r requirements/edx/base.txt # Install node and npm -RUN nodeenv /edx/app/edxapp/nodeenv --node=18.19.0 --prebuilt -RUN npm install -g npm@10.5.x +RUN nodeenv /edx/app/edxapp/nodeenv --node=20.15.1 --prebuilt +RUN npm install -g npm@10.7.x # This script is used by an npm post-install hook. # We copy it into the image now so that it will be available when we run `npm install` in the next step. @@ -178,8 +178,8 @@ FROM base as development RUN apt-get update && \ apt-get -y install --no-install-recommends \ - # wget is used in Makefile for common_constraints.txt - wget \ + # wget is used in Makefile for common_constraints.txt + wget \ && \ apt-get clean all && \ rm -rf /var/lib/apt/* diff --git a/Makefile b/Makefile index 15bab5df67a9..62681f6f3711 100644 --- a/Makefile +++ b/Makefile @@ -204,3 +204,37 @@ migrate: migrate-lms migrate-cms # Part of https://github.com/openedx/wg-developer-experience/issues/136 ubuntu-requirements: ## Install ubuntu 22.04 system packages needed for `pip install` to work on ubuntu. sudo apt install libmysqlclient-dev libxmlsec1-dev + +xsslint: ## check xss for quality issuest + python scripts/xsslint/xss_linter.py \ + --rule-totals \ + --config=scripts.xsslint_config \ + --thresholds=scripts/xsslint_thresholds.json + +pycodestyle: ## check python files for quality issues + pycodestyle . + +## Re-enable --lint flag when this issue https://github.com/openedx/edx-platform/issues/35775 is resolved +pii_check: ## check django models for pii annotations + DJANGO_SETTINGS_MODULE=cms.envs.test \ + code_annotations django_find_annotations \ + --config_file .pii_annotations.yml \ + --app_name cms \ + --coverage \ + --lint + + DJANGO_SETTINGS_MODULE=lms.envs.test \ + code_annotations django_find_annotations \ + --config_file .pii_annotations.yml \ + --app_name lms \ + --coverage \ + --lint + +check_keywords: ## check django models for reserve keywords + DJANGO_SETTINGS_MODULE=cms.envs.test \ + python manage.py cms check_reserved_keywords \ + --override_file db_keyword_overrides.yml + + DJANGO_SETTINGS_MODULE=lms.envs.test \ + python manage.py lms check_reserved_keywords \ + --override_file db_keyword_overrides.yml diff --git a/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py b/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py index a81d391b3f69..fa2a651f8a28 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/serializers/home.py @@ -66,6 +66,7 @@ class StudioHomeSerializer(serializers.Serializer): request_course_creator_url = serializers.CharField() rerun_creator_status = serializers.BooleanField() show_new_library_button = serializers.BooleanField() + show_new_library_v2_button = serializers.BooleanField() split_studio_home = serializers.BooleanField() studio_name = serializers.CharField() studio_short_name = serializers.CharField() diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/home.py b/cms/djangoapps/contentstore/rest_api/v1/views/home.py index 06433d9f42d5..3de536d78092 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/home.py @@ -66,6 +66,7 @@ def get(self, request: Request): "request_course_creator_url": "/request_course_creator", "rerun_creator_status": true, "show_new_library_button": true, + "show_new_library_v2_button": true, "split_studio_home": false, "studio_name": "Studio", "studio_short_name": "Studio", diff --git a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py index 73e3fe5ec3ad..a4a6909c5dcb 100644 --- a/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v1/views/tests/test_home.py @@ -2,7 +2,9 @@ Unit tests for home page view. """ import ddt +import pytz from collections import OrderedDict +from datetime import datetime, timedelta from django.conf import settings from django.test import override_settings from django.urls import reverse @@ -54,6 +56,7 @@ def setUp(self): "request_course_creator_url": "/request_course_creator", "rerun_creator_status": True, "show_new_library_button": True, + "show_new_library_v2_button": True, "split_studio_home": False, "studio_name": settings.STUDIO_NAME, "studio_short_name": settings.STUDIO_SHORT_NAME, @@ -100,12 +103,13 @@ class HomePageCoursesViewTest(CourseTestCase): def setUp(self): super().setUp() self.url = reverse("cms.djangoapps.contentstore:v1:courses") - CourseOverviewFactory.create( + self.course_overview = CourseOverviewFactory.create( id=self.course.id, org=self.course.org, display_name=self.course.display_name, display_number_with_default=self.course.number, ) + self.non_staff_client, _ = self.create_non_staff_authed_user_client() def test_home_page_response(self): """Check successful response content""" @@ -131,6 +135,7 @@ def test_home_page_response(self): print(response.data) self.assertDictEqual(expected_response, response.data) + @override_settings(FEATURES=FEATURES_WITH_HOME_PAGE_COURSE_V2_API) def test_home_page_response_with_api_v2(self): """Check successful response content with api v2 modifications. @@ -155,12 +160,88 @@ def test_home_page_response_with_api_v2(self): "in_process_course_actions": [], } - with override_settings(FEATURES=FEATURES_WITH_HOME_PAGE_COURSE_V2_API): - response = self.client.get(self.url) + response = self.client.get(self.url) self.assertEqual(response.status_code, status.HTTP_200_OK) self.assertDictEqual(expected_response, response.data) + @override_settings(FEATURES=FEATURES_WITH_HOME_PAGE_COURSE_V2_API) + @ddt.data( + ("active_only", "true", 2, 0), + ("archived_only", "true", 0, 1), + ("search", "sample", 1, 0), + ("search", "demo", 0, 1), + ("order", "org", 2, 1), + ("order", "display_name", 2, 1), + ("order", "number", 2, 1), + ("order", "run", 2, 1) + ) + @ddt.unpack + def test_filter_and_ordering_courses( + self, + filter_key, + filter_value, + expected_active_length, + expected_archived_length + ): + """Test home page with org filter and ordering for a staff user. + + The test creates an active/archived course, and then filters/orders them using the query parameters. + """ + archived_course_key = self.store.make_course_key("demo-org", "demo-number", "demo-run") + CourseOverviewFactory.create( + display_name="Course (Demo)", + id=archived_course_key, + org=archived_course_key.org, + end=(datetime.now() - timedelta(days=365)).replace(tzinfo=pytz.UTC), + ) + active_course_key = self.store.make_course_key("sample-org", "sample-number", "sample-run") + CourseOverviewFactory.create( + display_name="Course (Sample)", + id=active_course_key, + org=active_course_key.org, + ) + + response = self.client.get(self.url, {filter_key: filter_value}) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual(len(response.data["archived_courses"]), expected_archived_length) + self.assertEqual(len(response.data["courses"]), expected_active_length) + + @override_settings(FEATURES=FEATURES_WITH_HOME_PAGE_COURSE_V2_API) + @ddt.data( + ("active_only", "true"), + ("archived_only", "true"), + ("search", "sample"), + ("order", "org"), + ) + @ddt.unpack + def test_filter_and_ordering_no_courses_staff(self, filter_key, filter_value): + """Test home page with org filter and ordering when there are no courses for a staff user.""" + self.course_overview.delete() + + response = self.client.get(self.url, {filter_key: filter_value}) + + self.assertEqual(len(response.data["courses"]), 0) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + @override_settings(FEATURES=FEATURES_WITH_HOME_PAGE_COURSE_V2_API) + @ddt.data( + ("active_only", "true"), + ("archived_only", "true"), + ("search", "sample"), + ("order", "org"), + ) + @ddt.unpack + def test_home_page_response_no_courses_non_staff(self, filter_key, filter_value): + """Test home page with org filter and ordering when there are no courses for a non-staff user.""" + self.course_overview.delete() + + response = self.non_staff_client.get(self.url) + + self.assertEqual(len(response.data["courses"]), 0) + self.assertEqual(response.status_code, status.HTTP_200_OK) + @override_waffle_switch(ENABLE_GLOBAL_STAFF_OPTIMIZATION, True) def test_org_query_if_passed(self): """Test home page when org filter passed as a query param""" diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py index 6905de254f3e..e773e7f213c6 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_home.py @@ -45,6 +45,7 @@ def setUp(self): org=archived_course_key.org, end=(datetime.now() - timedelta(days=365)).replace(tzinfo=pytz.UTC), ) + self.non_staff_client, _ = self.create_non_staff_authed_user_client() def test_home_page_response(self): """Get list of courses available to the logged in user. @@ -247,3 +248,100 @@ def test_api_v2_is_disabled(self, mock_modulestore, mock_course_overview): self.assertEqual(response.status_code, status.HTTP_200_OK) mock_modulestore().get_course_summaries.assert_called_once() mock_course_overview.get_all_courses.assert_not_called() + + @ddt.data( + ("active_only", "true"), + ("archived_only", "true"), + ("search", "sample"), + ("order", "org"), + ("page", 1), + ) + @ddt.unpack + def test_if_empty_list_of_courses(self, query_param, value): + """Get list of courses when no courses are available. + + Expected result: + - An empty list of courses available to the logged in user. + """ + self.active_course.delete() + self.archived_course.delete() + + response = self.client.get(self.api_v2_url, {query_param: value}) + + self.assertEqual(len(response.data['results']['courses']), 0) + self.assertEqual(response.status_code, status.HTTP_200_OK) + + @override_settings(FEATURES=FEATURES_WITH_HOME_PAGE_COURSE_V2_API) + @ddt.data( + ("active_only", "true", 2, 0), + ("archived_only", "true", 0, 1), + ("search", "foo", 1, 0), + ("search", "demo", 0, 1), + ("order", "org", 2, 1), + ("order", "display_name", 2, 1), + ("order", "number", 2, 1), + ("order", "run", 2, 1) + ) + @ddt.unpack + def test_filter_and_ordering_courses( + self, + filter_key, + filter_value, + expected_active_length, + expected_archived_length + ): + """Get list of courses when filter and ordering are applied. + + This test creates two courses besides the default courses created in the setUp method. + Then filters and orders them based on the filter_key and filter_value passed as query parameters. + + Expected result: + - A list of courses available to the logged in user for the specified filter and order. + """ + archived_course_key = self.store.make_course_key("demo-org", "demo-number", "demo-run") + CourseOverviewFactory.create( + display_name="Course (Demo)", + id=archived_course_key, + org=archived_course_key.org, + end=(datetime.now() - timedelta(days=365)).replace(tzinfo=pytz.UTC), + ) + active_course_key = self.store.make_course_key("foo-org", "foo-number", "foo-run") + CourseOverviewFactory.create( + display_name="Course (Foo)", + id=active_course_key, + org=active_course_key.org, + ) + + response = self.client.get(self.api_v2_url, {filter_key: filter_value}) + + self.assertEqual(response.status_code, status.HTTP_200_OK) + self.assertEqual( + len([course for course in response.data["results"]["courses"] if course["is_active"]]), + expected_active_length + ) + self.assertEqual( + len([course for course in response.data["results"]["courses"] if not course["is_active"]]), + expected_archived_length + ) + + @ddt.data( + ("active_only", "true"), + ("archived_only", "true"), + ("search", "sample"), + ("order", "org"), + ("page", 1), + ) + @ddt.unpack + def test_if_empty_list_of_courses_non_staff(self, query_param, value): + """Get list of courses when no courses are available for non-staff users. + + Expected result: + - An empty list of courses available to the logged in user. + """ + self.active_course.delete() + self.archived_course.delete() + + response = self.non_staff_client.get(self.api_v2_url, {query_param: value}) + + self.assertEqual(len(response.data["results"]["courses"]), 0) + self.assertEqual(response.status_code, status.HTTP_200_OK) diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index c0f656ec7059..7023bcaefaf7 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -1552,6 +1552,9 @@ def get_library_context(request, request_is_json=False): from cms.djangoapps.contentstore.views.library import ( user_can_view_create_library_button, ) + from openedx.core.djangoapps.content_libraries.api import ( + user_can_create_library, + ) libraries = _accessible_libraries_iter(request.user) if libraries_v1_enabled() else [] data = { @@ -1565,6 +1568,7 @@ def get_library_context(request, request_is_json=False): 'courses': [], 'libraries_enabled': libraries_v1_enabled(), 'show_new_library_button': user_can_view_create_library_button(request.user) and request.user.is_active, + 'show_new_library_v2_button': user_can_create_library(request.user), 'user': request.user, 'request_course_creator_url': reverse('request_course_creator'), 'course_creator_status': _get_course_creator_status(request.user), @@ -1686,6 +1690,9 @@ def get_home_context(request, no_course=False): from cms.djangoapps.contentstore.views.library import ( user_can_view_create_library_button, ) + from openedx.core.djangoapps.content_libraries.api import ( + user_can_create_library, + ) active_courses = [] archived_courses = [] @@ -1714,6 +1721,7 @@ def get_home_context(request, no_course=False): 'taxonomy_list_mfe_url': get_taxonomy_list_url(), 'libraries': libraries, 'show_new_library_button': user_can_view_create_library_button(user), + 'show_new_library_v2_button': user_can_create_library(user), 'user': user, 'request_course_creator_url': reverse('request_course_creator'), 'course_creator_status': _get_course_creator_status(user), diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 9f6cfb7c430e..244804c3062b 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -15,6 +15,7 @@ from django.contrib.auth import get_user_model from django.contrib.auth.decorators import login_required from django.core.exceptions import FieldError, PermissionDenied, ValidationError as DjangoValidationError +from django.db.models import QuerySet from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseNotFound from django.shortcuts import redirect from django.urls import reverse @@ -575,6 +576,10 @@ def filter_ccx(course_access): if course_keys: courses_list = CourseOverview.get_all_courses(filter_={'id__in': course_keys}) + else: + # If no course keys are found for the current user, then return without filtering + # or ordering the courses list. + return courses_list, [] search_query, order, active_only, archived_only = get_query_params_if_present(request) courses_list = get_filtered_and_ordered_courses( @@ -588,7 +593,11 @@ def filter_ccx(course_access): return courses_list, [] -def get_courses_by_status(active_only, archived_only, course_overviews): +def get_courses_by_status( + active_only: bool, + archived_only: bool, + course_overviews: QuerySet[CourseOverview] +) -> QuerySet[CourseOverview]: """ Return course overviews based on a base queryset filtered by a status. @@ -602,7 +611,10 @@ def get_courses_by_status(active_only, archived_only, course_overviews): return CourseOverview.get_courses_by_status(active_only, archived_only, course_overviews) -def get_courses_by_search_query(search_query, course_overviews): +def get_courses_by_search_query( + search_query: str | None, + course_overviews: QuerySet[CourseOverview] +) -> QuerySet[CourseOverview]: """Return course overviews based on a base queryset filtered by a search query. Args: @@ -614,7 +626,10 @@ def get_courses_by_search_query(search_query, course_overviews): return CourseOverview.get_courses_matching_query(search_query, course_overviews=course_overviews) -def get_courses_order_by(order_query, course_overviews): +def get_courses_order_by( + order_query: str | None, + course_overviews: QuerySet[CourseOverview] +) -> QuerySet[CourseOverview]: """Return course overviews based on a base queryset ordered by a query. Args: diff --git a/cms/djangoapps/contentstore/views/tests/test_block.py b/cms/djangoapps/contentstore/views/tests/test_block.py index a6fefe5f554c..1190eb239518 100644 --- a/cms/djangoapps/contentstore/views/tests/test_block.py +++ b/cms/djangoapps/contentstore/views/tests/test_block.py @@ -312,12 +312,12 @@ def test_split_test(self): resp = self.create_xblock( parent_usage_key=split_test_usage_key, category="html", - boilerplate="zooming_image.yaml", + boilerplate="latex_html.yaml", ) self.assertEqual(resp.status_code, 200) html, __ = self._get_container_preview(split_test_usage_key) self.assertIn("Announcement", html) - self.assertIn("Zooming", html) + self.assertIn("LaTeX", html) def test_split_test_edited(self): """ diff --git a/cms/djangoapps/contentstore/views/tests/test_course_index.py b/cms/djangoapps/contentstore/views/tests/test_course_index.py index c3dcfe5305b7..a49273928062 100644 --- a/cms/djangoapps/contentstore/views/tests/test_course_index.py +++ b/cms/djangoapps/contentstore/views/tests/test_course_index.py @@ -433,13 +433,13 @@ def check_index_page(self, separate_archived_courses, org): @override_settings(FEATURES=FEATURES_WITHOUT_HOME_PAGE_COURSE_V2_API) @ddt.data( # Staff user has course staff access - (True, 'staff', None, 0, 21), - (False, 'staff', None, 0, 21), + (True, 'staff', None, 0, 23), + (False, 'staff', None, 0, 23), # Base user has global staff access - (True, 'user', ORG, 2, 21), - (False, 'user', ORG, 2, 21), - (True, 'user', None, 2, 21), - (False, 'user', None, 2, 21), + (True, 'user', ORG, 2, 23), + (False, 'user', ORG, 2, 23), + (True, 'user', None, 2, 23), + (False, 'user', None, 2, 23), ) @ddt.unpack def test_separate_archived_courses(self, separate_archived_courses, username, org, mongo_queries, sql_queries): @@ -464,13 +464,13 @@ def test_separate_archived_courses(self, separate_archived_courses, username, or @override_settings(FEATURES=FEATURES_WITH_HOME_PAGE_COURSE_V2_API) @ddt.data( # Staff user has course staff access - (True, 'staff', None, 0, 21), - (False, 'staff', None, 0, 21), + (True, 'staff', None, 0, 23), + (False, 'staff', None, 0, 23), # Base user has global staff access - (True, 'user', ORG, 0, 21), - (False, 'user', ORG, 0, 21), - (True, 'user', None, 0, 21), - (False, 'user', None, 0, 21), + (True, 'user', ORG, 0, 23), + (False, 'user', ORG, 0, 23), + (True, 'user', None, 0, 23), + (False, 'user', None, 0, 23), ) @ddt.unpack def test_separate_archived_courses_with_home_page_course_v2_api( diff --git a/common/djangoapps/static_replace/__init__.py b/common/djangoapps/static_replace/__init__.py index 310ad8343242..cfab4dde8375 100644 --- a/common/djangoapps/static_replace/__init__.py +++ b/common/djangoapps/static_replace/__init__.py @@ -11,7 +11,7 @@ from xmodule.contentstore.content import StaticContent log = logging.getLogger(__name__) -XBLOCK_STATIC_RESOURCE_PREFIX = '/static/xblock' +XBLOCK_STATIC_RESOURCE_PREFIX = '/static/xblock/' def _url_replace_regex(prefix): diff --git a/common/djangoapps/static_replace/test/test_static_replace.py b/common/djangoapps/static_replace/test/test_static_replace.py index 1bdb770cad31..c71f8fa15c6a 100644 --- a/common/djangoapps/static_replace/test/test_static_replace.py +++ b/common/djangoapps/static_replace/test/test_static_replace.py @@ -86,6 +86,16 @@ def processor(__, prefix, quote, rest): # pylint: disable=redefined-outer-name assert process_static_urls(STATIC_SOURCE, processor) == '"test/static/file.png"' +def test_process_url_no_match_starts_with_xblock(): + def processor(original, prefix, quote, rest): # pylint: disable=unused-argument, redefined-outer-name + return quote + 'test' + prefix + rest + quote + assert process_static_urls( + '"/static/xblock-file.png"', + processor, + data_dir=DATA_DIRECTORY + ) == '"test/static/xblock-file.png"' + + @patch('django.http.HttpRequest', autospec=True) def test_static_urls(mock_request): mock_request.build_absolute_uri = lambda url: 'http://' + url diff --git a/docs/concepts/extension_points.rst b/docs/concepts/extension_points.rst index d4e802baec0e..3136aa8057c2 100644 --- a/docs/concepts/extension_points.rst +++ b/docs/concepts/extension_points.rst @@ -139,10 +139,10 @@ Here are the different integration points that python plugins can use: - This decorator allows overriding any function or method by pointing to an alternative implementation in settings. Read the |pluggable_override docstring|_ to learn more. * - Open edX Events - Adopt, Stable - - Events are part of the greater Hooks Extension Framework for open extension of edx-platform. Events are a stable way for plugin developers to react to learner or author events. They are defined by a `separate events library`_ that developers can include in their requirements to develop and test the code without creating a dependency on this large repo. For more information see the `hooks guide`_. + - Events are part of the greater Hooks Extension Framework for open extension of edx-platform. Events are a stable way for plugin developers to react to learner or author events. They are defined by a `separate events library`_ that developers can include in their requirements to develop and test the code without creating a dependency on this large repo. For more information see the `Hooks Extension Framework docs`_ or for more detailed documentation about Open edX Events, see the `Open edX Events documentation`_. * - Open edX Filters - Adopt, Stable - - Filters are also part of Hooks Extension Framework for open extension of edx-platform. Filters are a flexible way for plugin developers to modify learner or author application flows. They are defined by a `separate filters library`_ that developers can include in their requirements to develop and test the code without creating a dependency on this large repo. For more information see the `hooks guide`_. + - Filters are also part of Hooks Extension Framework for open extension of edx-platform. Filters are a flexible way for plugin developers to modify learner or author application flows. They are defined by a `separate filters library`_ that developers can include in their requirements to develop and test the code without creating a dependency on this large repo. For more information see the `Hooks Extension Framework docs`_ or for more detailed documentation about Open edX Filters, see the `Open edX Filters documentation`_. .. _Application: https://docs.djangoproject.com/en/3.0/ref/applications/ .. _Django app plugin documentation: https://github.com/openedx/edx-platform/blob/master/openedx/core/djangoapps/plugins/README.rst @@ -159,7 +159,9 @@ Here are the different integration points that python plugins can use: .. _pluggable_override docstring: https://github.com/openedx/edx-django-utils/blob/master/edx_django_utils/plugins/pluggable_override.py .. _separate events library: https://github.com/eduNEXT/openedx-events/ .. _separate filters library: https://github.com/eduNEXT/openedx-filters/ -.. _hooks guide: https://github.com/openedx/edx-platform/blob/master/docs/guides/hooks/index.rst +.. _Hooks Extension Framework docs: https://docs.openedx.org/en/latest/developers/concepts/hooks_extension_framework.html +.. _Open edX Events documentation: https://docs.openedx.org/projects/openedx-events/en/latest/ +.. _Open edX Filters documentation: https://docs.openedx.org/projects/openedx-filters/en/latest/ Platform Look & Feel ==================== diff --git a/docs/conf.py b/docs/conf.py index 604ca3755429..ec416f1c19e6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -67,19 +67,22 @@ 'sphinx_design', 'code_annotations.contrib.sphinx.extensions.featuretoggles', 'code_annotations.contrib.sphinx.extensions.settings', - 'autoapi.extension', + # 'autoapi.extension', # Temporarily disabled + 'sphinx_reredirects', ] -autoapi_type = 'python' -autoapi_dirs = ['../lms', '../openedx'] - -autoapi_ignore = [ - '*/migrations/*', - '*/tests/*', - '*.pyc', - '__init__.py', - '**/xblock_serializer/data.py', -] +# Temporarily disabling autoapi_dirs and the AutoAPI extension due to performance issues. +# This will unblock ReadTheDocs builds and will be revisited for optimization. +# autoapi_type = 'python' +# autoapi_dirs = ['../lms/djangoapps', '../openedx/core/djangoapps', "../openedx/features"] +# +# autoapi_ignore = [ +# '*/migrations/*', +# '*/tests/*', +# '*.pyc', +# '__init__.py', +# '**/xblock_serializer/data.py', +# ] # Rediraffe related settings. rediraffe_redirects = "redirects.txt" @@ -285,7 +288,7 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - 'django': ('https://docs.djangoproject.com/en/1.11/', 'https://docs.djangoproject.com/en/1.11/_objects/'), + 'django': ('https://docs.djangoproject.com/en/4.2/', 'https://docs.djangoproject.com/en/4.2/_objects/'), } # Start building a map of the directories relative to the repository root to @@ -302,6 +305,16 @@ # 'xmodule': 'references/docstrings/xmodule', } +# Mapping permanently moved pages to appropriate new location outside of edx-platform +# with by sphinx-reredirects extension redirects. +# More information: https://documatt.com/sphinx-reredirects/usage.html + +redirects = { + 'hooks/events': 'https://docs.openedx.org/projects/openedx-events/en/latest/', + 'hooks/filters': 'https://docs.openedx.org/projects/openedx-filters/en/latest/', + 'hooks/index': 'https://docs.openedx.org/en/latest/developers/concepts/hooks_extension_framework.html', +} + def update_settings_module(service='lms'): """ diff --git a/docs/hooks/events.rst b/docs/hooks/events.rst deleted file mode 100644 index bccb98e56a42..000000000000 --- a/docs/hooks/events.rst +++ /dev/null @@ -1,261 +0,0 @@ -Open edX Events -=============== - -How to use ----------- - -Using openedx-events in your code is very straight forward. We can consider the -two possible cases, sending or receiving an event. - - -Receiving events -^^^^^^^^^^^^^^^^ - -This is one of the most common use cases for plugins. The edx-platform will send -an event and you want to react to it in your plugin. - -For this you need to: - -1. Include openedx-events in your dependencies. -2. Connect your receiver functions to the signals being sent. - -Connecting signals can be done using regular django syntax: - -.. code-block:: python - - from openedx_events.learning.signals import SESSION_LOGIN_COMPLETED - - @receiver(SESSION_LOGIN_COMPLETED) - # your receiver function here - - -Or at the apps.py - -.. code-block:: python - - "signals_config": { - "lms.djangoapp": { - "relative_path": "your_module_name", - "receivers": [ - { - "receiver_func_name": "your_receiver_function", - "signal_path": "openedx_events.learning.signals.SESSION_LOGIN_COMPLETED", - }, - ], - } - }, - -In case you are listening to an event in the edx-platform repo, you can directly -use the django syntax since the apps.py method will not be available without the -plugin. - - -Sending events -^^^^^^^^^^^^^^ - -Sending events requires you to import both the event definition as well as the -attr data classes that encapsulate the event data. - -.. code-block:: python - - from openedx_events.learning.data import UserData, UserPersonalData - from openedx_events.learning.signals import STUDENT_REGISTRATION_COMPLETED - - STUDENT_REGISTRATION_COMPLETED.send_event( - user=UserData( - pii=UserPersonalData( - username=user.username, - email=user.email, - name=user.profile.name, - ), - id=user.id, - is_active=user.is_active, - ), - ) - -You can do this both from the edx-platform code as well as from an openedx -plugin. - - -Testing events -^^^^^^^^^^^^^^ - -Testing your code in CI, specially for plugins is now possible without having to -import the complete edx-platform as a dependency. - -To test your functions you need to include the openedx-events library in your -testing dependencies and make the signal connection in your test case. - -.. code-block:: python - - from openedx_events.learning.signals import STUDENT_REGISTRATION_COMPLETED - - def test_your_receiver(self): - STUDENT_REGISTRATION_COMPLETED.connect(your_function) - STUDENT_REGISTRATION_COMPLETED.send_event( - user=UserData( - pii=UserPersonalData( - username='test_username', - email='test_email@example.com', - name='test_name', - ), - id=1, - is_active=True, - ), - ) - - # run your assertions - - -Changes in the openedx-events library that are not compatible with your code -should break this kind of test in CI and let you know you need to upgrade your -code. - - -Live example -^^^^^^^^^^^^ - -For a complete and detailed example you can see the `openedx-events-2-zapier`_ -plugin. This is a fully functional plugin that connects to -``STUDENT_REGISTRATION_COMPLETED`` and ``COURSE_ENROLLMENT_CREATED`` and sends -the relevant information to zapier.com using a webhook. - -.. _openedx-events-2-zapier: https://github.com/eduNEXT/openedx-events-2-zapier - - -Index of Events ------------------ - -This list contains the events currently being sent by edx-platform. The provided -links target both the definition of the event in the openedx-events library as -well as the trigger location in this same repository. - - -Learning Events -^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 35 50 20 - - * - *Name* - - *Type* - - *Date added* - - * - `STUDENT_REGISTRATION_COMPLETED `_ - - org.openedx.learning.student.registration.completed.v1 - - `2022-06-14 `_ - - * - `SESSION_LOGIN_COMPLETED `_ - - org.openedx.learning.auth.session.login.completed.v1 - - `2022-06-14 `_ - - * - `COURSE_ENROLLMENT_CREATED `_ - - org.openedx.learning.course.enrollment.created.v1 - - `2022-06-14 `_ - - * - `COURSE_ENROLLMENT_CHANGED `_ - - org.openedx.learning.course.enrollment.changed.v1 - - `2022-06-14 `_ - - * - `COURSE_UNENROLLMENT_COMPLETED `_ - - org.openedx.learning.course.unenrollment.completed.v1 - - `2022-06-14 `_ - - * - `CERTIFICATE_CREATED `_ - - org.openedx.learning.certificate.created.v1 - - `2022-06-14 `_ - - * - `CERTIFICATE_CHANGED `_ - - org.openedx.learning.certificate.changed.v1 - - `2022-06-14 `_ - - * - `CERTIFICATE_REVOKED `_ - - org.openedx.learning.certificate.revoked.v1 - - `2022-06-14 `_ - - * - `COHORT_MEMBERSHIP_CHANGED `_ - - org.openedx.learning.cohort_membership.changed.v1 - - `2022-06-14 `_ - - * - `COURSE_DISCUSSIONS_CHANGED `_ - - org.openedx.learning.discussions.configuration.changed.v1 - - `2022-06-14 `_ - - -Content Authoring Events -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :widths: 35 50 20 - - * - *Name* - - *Type* - - *Date added* - - * - `COURSE_CATALOG_INFO_CHANGED `_ - - org.openedx.content_authoring.course.catalog_info.changed.v1 - - `2022-08-24 `_ - - * - `XBLOCK_PUBLISHED `_ - - org.openedx.content_authoring.xblock.published.v1 - - `2022-12-06 `_ - - * - `XBLOCK_DELETED `_ - - org.openedx.content_authoring.xblock.deleted.v1 - - `2022-12-06 `_ - - * - `XBLOCK_DUPLICATED `_ - - org.openedx.content_authoring.xblock.duplicated.v1 - - `2022-12-06 `_ - - * - `XBLOCK_CREATED `_ - - org.openedx.content_authoring.xblock.created.v1 - - 2023-07-20 - - * - `XBLOCK_UPDATED `_ - - org.openedx.content_authoring.xblock.updated.v1 - - 2023-07-20 - - * - `COURSE_CREATED `_ - - org.openedx.content_authoring.course.created.v1 - - 2023-07-20 - - * - `CONTENT_LIBRARY_CREATED `_ - - org.openedx.content_authoring.content_library.created.v1 - - 2023-07-20 - - * - `CONTENT_LIBRARY_UPDATED `_ - - org.openedx.content_authoring.content_library.updated.v1 - - 2023-07-20 - - * - `CONTENT_LIBRARY_DELETED `_ - - org.openedx.content_authoring.content_library.deleted.v1 - - 2023-07-20 - - * - `LIBRARY_BLOCK_CREATED `_ - - org.openedx.content_authoring.library_block.created.v1 - - 2023-07-20 - - * - `LIBRARY_BLOCK_UPDATED `_ - - org.openedx.content_authoring.library_block.updated.v1 - - 2023-07-20 - - * - `LIBRARY_BLOCK_DELETED `_ - - org.openedx.content_authoring.library_block.deleted.v1 - - 2023-07-20 - - * - `LIBRARY_COLLECTION_CREATED `_ - - org.openedx.content_authoring.content_library.collection.created.v1 - - 2024-08-23 - - * - `LIBRARY_COLLECTION_UPDATED `_ - - org.openedx.content_authoring.content_library.collection.updated.v1 - - 2024-08-23 - - * - `LIBRARY_COLLECTION_DELETED `_ - - org.openedx.content_authoring.content_library.collection.deleted.v1 - - 2024-08-23 - - * - `CONTENT_OBJECT_ASSOCIATIONS_CHANGED `_ - - org.openedx.content_authoring.content.object.associations.changed.v1 - - 2024-09-06 diff --git a/docs/hooks/filters.rst b/docs/hooks/filters.rst deleted file mode 100644 index b2ce68fc147d..000000000000 --- a/docs/hooks/filters.rst +++ /dev/null @@ -1,191 +0,0 @@ -Open edX Filters -================ - -How to use ----------- - -Using openedx-filters in your code is very straight forward. We can consider the -two possible cases: - -Configuring a filter -^^^^^^^^^^^^^^^^^^^^ - -Implement pipeline steps -************************ - -Let's say you want to consult student's information with a third party service -before generating the students certificate. This is a common use case for filters, -where the functions part of the filter's pipeline will perform the consulting tasks and -decide the execution flow for the application. These functions are the pipeline steps, -and can be implemented in an installable Python library: - -.. code-block:: python - - # Step implementation taken from openedx-filters-samples plugin - from openedx_filters import PipelineStep - from openedx_filters.learning.filters import CertificateCreationRequested - - class StopCertificateCreation(PipelineStep): - - def run_filter(self, user, course_id, mode, status): - # Consult third party service and check if continue - # ... - # User not in third party service, denied certificate generation - raise CertificateCreationRequested.PreventCertificateCreation( - "You can't generate a certificate from this site." - ) - -There's two key components to the implementation: - -1. The filter step must be a subclass of ``PipelineStep``. - -2. The ``run_filter`` signature must match the filters definition, eg., -the previous step matches the method's definition in CertificateCreationRequested. - -Attach/hook pipeline to filter -****************************** - -After implementing the pipeline steps, we have to tell the certificate creation -filter to execute our pipeline. - -.. code-block:: python - - OPEN_EDX_FILTERS_CONFIG = { - "org.openedx.learning.certificate.creation.requested.v1": { - "fail_silently": False, - "pipeline": [ - "openedx_filters_samples.samples.pipeline.StopCertificateCreation" - ] - }, - } - -Triggering a filter -^^^^^^^^^^^^^^^^^^^ - -In order to execute a filter in your own plugin/library, you must install the -plugin where the steps are implemented and also, ``openedx-filters``. - -.. code-block:: python - - # Code taken from lms/djangoapps/certificates/generation_handler.py - from openedx_filters.learning.filters import CertificateCreationRequested - - try: - self.user, self.course_id, self.mode, self.status = CertificateCreationRequested.run_filter( - user=self.user, course_id=self.course_id, mode=self.mode, status=self.status, - ) - except CertificateCreationRequested.PreventCertificateCreation as exc: - raise CertificateGenerationNotAllowed(str(exc)) from exc - -Testing filters' steps -^^^^^^^^^^^^^^^^^^^^^^ - -It's pretty straightforward to test your pipeline steps, you'll need to include the -``openedx-filters`` library in your testing dependencies and configure them in your test case. - -.. code-block:: python - - from openedx_filters.learning.filters import CertificateCreationRequested - - @override_settings( - OPEN_EDX_FILTERS_CONFIG={ - "org.openedx.learning.certificate.creation.requested.v1": { - "fail_silently": False, - "pipeline": [ - "openedx_filters_samples.samples.pipeline.StopCertificateCreation" - ] - } - } - ) - def test_certificate_creation_requested_filter(self): - """ - Test filter triggered before the certificate creation process starts. - - Expected results: - - The pipeline step configured for the filter raises PreventCertificateCreation - when the conditions are met. - """ - with self.assertRaises(CertificateCreationRequested.PreventCertificateCreation): - CertificateCreationRequested.run_filter( - user=self.user, course_key=self.course_key, mode="audit", - ) - - # run your assertions - -Changes in the ``openedx-filters`` library that are not compatible with your code -should break this kind of test in CI and let you know you need to upgrade your code. -The main limitation while testing filters' steps it's their arguments, as they are edxapp -memory objects, but that can be solved in CI using Python mocks. - -Live example -^^^^^^^^^^^^ - -For filter steps samples you can visit the `openedx-filters-samples`_ plugin, where -you can find minimal steps exemplifying the different ways on how to use -``openedx-filters``. - -.. _openedx-filters-samples: https://github.com/eduNEXT/openedx-filters-samples - - -Index of Filters ------------------ - -This list contains the filters currently being executed by edx-platform. The provided -links target both the definition of the filter in the openedx-filters library as -well as the trigger location in this same repository. - - -.. list-table:: - :widths: 35 50 20 - - * - *Name* - - *Type* - - *Date added* - - * - `StudentRegistrationRequested `_ - - org.openedx.learning.student.registration.requested.v1 - - `2022-06-14 `_ - - * - `StudentLoginRequested `_ - - org.openedx.learning.student.login.requested.v1 - - `2022-06-14 `_ - - * - `CourseEnrollmentStarted `_ - - org.openedx.learning.course.enrollment.started.v1 - - `2022-06-14 `_ - - * - `CourseUnenrollmentStarted `_ - - org.openedx.learning.course.unenrollment.started.v1 - - `2022-06-14 `_ - - * - `CertificateCreationRequested `_ - - org.openedx.learning.certificate.creation.requested.v1 - - `2022-06-14 `_ - - * - `CertificateRenderStarted `_ - - org.openedx.learning.certificate.render.started.v1 - - `2022-06-14 `_ - - * - `CohortChangeRequested `_ - - org.openedx.learning.cohort.change.requested.v1 - - `2022-06-14 `_ - - * - `CohortAssignmentRequested `_ - - org.openedx.learning.cohort.assignment.requested.v1 - - `2022-06-14 `_ - - * - `CourseAboutRenderStarted `_ - - org.openedx.learning.course_about.render.started.v1 - - `2022-06-14 `_ - - * - `DashboardRenderStarted `_ - - org.openedx.learning.dashboard.render.started.v1 - - `2022-06-14 `_ - - * - `VerticalBlockChildRenderStarted `_ - - org.openedx.learning.veritical_block_child.render.started.v1 - - `2022-08-18 `_ - - * - `VerticalBlockRenderCompleted `_ - - org.openedx.learning.veritical_block.render.completed.v1 - - `2022-02-18 `_ diff --git a/docs/hooks/index.rst b/docs/hooks/index.rst deleted file mode 100644 index 99cb25133cd2..000000000000 --- a/docs/hooks/index.rst +++ /dev/null @@ -1,50 +0,0 @@ -Open edX Hooks Extension Framework -================================== - -To sustain the growth of the Open edX ecosystem, the business rules of the -platform must be open for extension following the open-closed principle. This -framework allows developers to do just that without needing to fork and modify -the main edx-platform repository. - - -Context -------- - -Hooks are predefined places in the edx-platform core where externally defined -functions can take place. In some cases, those functions can alter what the user -sees or experiences in the platform. Other cases are informative only. All cases -are meant to be extended using Open edX plugins and configuration. - -Hooks can be of two types, events and filters. Events are in essence signals, in -that they are sent in specific application places and whose listeners can extend -functionality. On the other hand Filters are passed data and can act on it -before this data is put back in the original application flow. In order to allow -extension developers to use the Events and Filters definitions on their plugins, -both kinds of hooks are defined in lightweight external libraries. - -* openedx-filters (`guide <./filters.rst>`_, `source code `_) -* openedx-events (`guide <./events.rst>`_, `source code `_) - -Hooks are designed with stability in mind. The main goal is that developers can -use them to change the functionality of the platform as needed and still be able -to migrate to newer open releases with very little to no development effort. In -the case of the events, this is detailed in the `versioning ADR`_ and the -`payload ADR`_. - -A longer description of the framework and it's history can be found in `OEP 50`_. - -.. _OEP 50: https://open-edx-proposals.readthedocs.io/en/latest/oep-0050-hooks-extension-framework.html -.. _versioning ADR: https://github.com/eduNEXT/openedx-events/blob/main/docs/decisions/0002-events-naming-and-versioning.rst -.. _payload ADR: https://github.com/eduNEXT/openedx-events/blob/main/docs/decisions/0003-events-payload.rst - -On the technical side events are implemented through django signals which makes -them run in the same python process as the lms or cms. Furthermore, events block -the running process. Listeners of an event are encouraged to monitor the -performance or use alternative arch patterns such as receiving the event and -defer to launching async tasks than do the slow processing. - -On the other hand, filters are implemented using a pipeline mechanism, that executes -a list of functions called ``steps`` configured through Django settings. Each -pipeline step receives a dictionary with data, process it and returns an output. During -this process, they can alter the application execution flow by halting the process -or modifying their input arguments. diff --git a/docs/index.rst b/docs/index.rst index 8d89969398bc..190ba12db906 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,6 +20,7 @@ locations. .. _Developer Documentation Index: https://openedx.atlassian.net/wiki/spaces/DOC/overview .. _Open edX Development space: https://openedx.atlassian.net/wiki/spaces/COMM/overview .. _Open edX ReadTheDocs: http://docs.edx.org/ +.. _Hooks Extensions Framework: https://docs.openedx.org/en/latest/developers/concepts/hooks_extension_framework.html .. toctree:: :maxdepth: 1 @@ -32,7 +33,6 @@ locations. how-tos/index references/index concepts/index - hooks/index extensions/tinymce_plugins .. grid:: 1 2 2 2 @@ -80,14 +80,16 @@ locations. :class-card: sd-shadow-md sd-p-2 :class-footer: sd-border-0 - * :doc:`hooks/index` + * `Hooks Extensions Framework`_ * :doc:`extensions/tinymce_plugins` +++ - .. button-ref:: hooks/index + .. button-link:: https://docs.openedx.org/en/latest/developers/concepts/hooks_extension_framework.html :color: primary :outline: :expand: + Hooks Extensions Framework + Change History ************** diff --git a/docs/lms-openapi.yaml b/docs/lms-openapi.yaml index 4d318f8a2b62..ad3a4fc40db4 100644 --- a/docs/lms-openapi.yaml +++ b/docs/lms-openapi.yaml @@ -5847,7 +5847,7 @@ paths: operationId: grades_v1_submission_history_read description: |- Get submission history details. This submission history is related to only - ProblemBlock and it doesn't support LibraryContentBlock or ContentLibraries + ProblemBlock and it doesn't support LegacyLibraryContentBlock or ContentLibraries as of now. **Usecases**: @@ -7047,6 +7047,25 @@ paths: in: path required: true type: string + /mobile/{api_version}/users/{username}/enrollments_status/: + get: + operationId: mobile_users_enrollments_status_list + description: Gets user's enrollments status. + parameters: [] + responses: + '200': + description: '' + tags: + - mobile + parameters: + - name: api_version + in: path + required: true + type: string + - name: username + in: path + required: true + type: string /notifications/: get: operationId: notifications_list @@ -9868,7 +9887,94 @@ paths: required: true type: string pattern: ^[a-zA-Z0-9\-_]*$ - /xblock/v2/xblocks/{usage_key_str}/: + /xblock/v2/xblocks/{usage_key}/: + get: + operationId: xblock_v2_xblocks_read + summary: Get metadata about the specified block. + description: |- + Accepts the following query parameters: + + * "include": a comma-separated list of keys to include. + Valid keys are "index_dictionary" and "student_view_data". + parameters: [] + responses: + '200': + description: '' + tags: + - xblock + parameters: + - name: usage_key + in: path + required: true + type: string + /xblock/v2/xblocks/{usage_key}/fields/: + get: + operationId: xblock_v2_xblocks_fields_list + description: retrieves the xblock, returning display_name, data, and metadata + parameters: [] + responses: + '200': + description: '' + tags: + - xblock + post: + operationId: xblock_v2_xblocks_fields_create + description: edits the xblock, saving changes to data and metadata only (display_name + included in metadata) + parameters: [] + responses: + '201': + description: '' + tags: + - xblock + parameters: + - name: usage_key + in: path + required: true + type: string + /xblock/v2/xblocks/{usage_key}/handler_url/{handler_name}/: + get: + operationId: xblock_v2_xblocks_handler_url_read + summary: |- + Get an absolute URL which can be used (without any authentication) to call + the given XBlock handler. + description: The URL will expire but is guaranteed to be valid for a minimum + of 2 days. + parameters: [] + responses: + '200': + description: '' + tags: + - xblock + parameters: + - name: usage_key + in: path + required: true + type: string + - name: handler_name + in: path + required: true + type: string + /xblock/v2/xblocks/{usage_key}/view/{view_name}/: + get: + operationId: xblock_v2_xblocks_view_read + description: Get the HTML, JS, and CSS needed to render the given XBlock. + parameters: [] + responses: + '200': + description: '' + tags: + - xblock + parameters: + - name: usage_key + in: path + required: true + type: string + - name: view_name + in: path + required: true + type: string + /xblock/v2/xblocks/{usage_key}@{version}/: get: operationId: xblock_v2_xblocks_read summary: Get metadata about the specified block. @@ -9884,11 +9990,15 @@ paths: tags: - xblock parameters: - - name: usage_key_str + - name: usage_key in: path required: true type: string - /xblock/v2/xblocks/{usage_key_str}/fields/: + - name: version + in: path + required: true + type: string + /xblock/v2/xblocks/{usage_key}@{version}/fields/: get: operationId: xblock_v2_xblocks_fields_list description: retrieves the xblock, returning display_name, data, and metadata @@ -9909,11 +10019,15 @@ paths: tags: - xblock parameters: - - name: usage_key_str + - name: usage_key in: path required: true type: string - /xblock/v2/xblocks/{usage_key_str}/handler_url/{handler_name}/: + - name: version + in: path + required: true + type: string + /xblock/v2/xblocks/{usage_key}@{version}/handler_url/{handler_name}/: get: operationId: xblock_v2_xblocks_handler_url_read summary: |- @@ -9928,7 +10042,11 @@ paths: tags: - xblock parameters: - - name: usage_key_str + - name: usage_key + in: path + required: true + type: string + - name: version in: path required: true type: string @@ -9936,7 +10054,7 @@ paths: in: path required: true type: string - /xblock/v2/xblocks/{usage_key_str}/view/{view_name}/: + /xblock/v2/xblocks/{usage_key}@{version}/view/{view_name}/: get: operationId: xblock_v2_xblocks_view_read description: Get the HTML, JS, and CSS needed to render the given XBlock. @@ -9947,7 +10065,11 @@ paths: tags: - xblock parameters: - - name: usage_key_str + - name: usage_key + in: path + required: true + type: string + - name: version in: path required: true type: string @@ -10158,6 +10280,7 @@ definitions: - can_view_certificate - course_modes - is_new_discussion_sidebar_view_enabled + - has_course_author_access type: object properties: can_show_upgrade_sock: @@ -10237,6 +10360,9 @@ definitions: is_new_discussion_sidebar_view_enabled: title: Is new discussion sidebar view enabled type: boolean + has_course_author_access: + title: Has course author access + type: boolean DateSummary: required: - complete diff --git a/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py b/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py index b49d79976c06..eea03bd79455 100644 --- a/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py +++ b/lms/djangoapps/course_goals/management/commands/goal_reminder_email.py @@ -119,7 +119,11 @@ def send_ace_message(goal, session_id): with emulate_http_request(site, user): try: + start_time = datetime.now() ace.send(msg) + end_time = datetime.now() + log.info(f"Goal Reminder for {user.id} for course {goal.course_key} sent in {end_time - start_time} " + f"using {'SES' if is_ses_enabled else 'others'}") except Exception as exc: # pylint: disable=broad-except log.error(f"Goal Reminder for {user.id} for course {goal.course_key} could not send: {exc}") tracker.emit( diff --git a/lms/templates/vert_module.html b/lms/templates/vert_module.html index 7df5b8b5b46e..909e7db829c1 100644 --- a/lms/templates/vert_module.html +++ b/lms/templates/vert_module.html @@ -62,7 +62,7 @@

${unit_title}

% for idx, item in enumerate(items): % if item['content']: -
+
${HTML(item['content'])}
%endif diff --git a/openedx/core/djangoapps/content/block_structure/models.py b/openedx/core/djangoapps/content/block_structure/models.py index c2837e1a77f0..b3a8439ca169 100644 --- a/openedx/core/djangoapps/content/block_structure/models.py +++ b/openedx/core/djangoapps/content/block_structure/models.py @@ -41,8 +41,7 @@ def _directory_name(data_usage_key): # .. setting_description: Specifies the path in storage where block structures would be saved, # for storage-backed block structure cache. # For more information, check https://github.com/openedx/edx-platform/pull/14571. - # .. setting_warnings: Depends on `BLOCK_STRUCTURES_SETTINGS['STORAGE_CLASS']` and on - # `block_structure.storage_backing_for_cache`. + # .. setting_warnings: Depends on `BLOCK_STRUCTURES_SETTINGS['STORAGE_CLASS']` directory_prefix = settings.BLOCK_STRUCTURES_SETTINGS.get('DIRECTORY_PREFIX', '') # replace any '/' in the usage key so they aren't interpreted diff --git a/openedx/core/djangoapps/content/search/api.py b/openedx/core/djangoapps/content/search/api.py index b1d224b411e4..a18c55bd3d22 100644 --- a/openedx/core/djangoapps/content/search/api.py +++ b/openedx/core/djangoapps/content/search/api.py @@ -5,7 +5,7 @@ import logging import time -from contextlib import contextmanager +from contextlib import contextmanager, nullcontext from datetime import datetime, timedelta, timezone from functools import wraps from typing import Callable, Generator @@ -24,7 +24,14 @@ from rest_framework.request import Request from common.djangoapps.student.role_helpers import get_course_roles from openedx.core.djangoapps.content.course_overviews.models import CourseOverview -from openedx.core.djangoapps.content.search.models import get_access_ids_for_request +from openedx.core.djangoapps.content.search.models import get_access_ids_for_request, IncrementalIndexCompleted +from openedx.core.djangoapps.content.search.index_config import ( + INDEX_DISTINCT_ATTRIBUTE, + INDEX_FILTERABLE_ATTRIBUTES, + INDEX_SEARCHABLE_ATTRIBUTES, + INDEX_SORTABLE_ATTRIBUTES, + INDEX_RANKING_RULES, +) from openedx.core.djangoapps.content_libraries import api as lib_api from xmodule.modulestore.django import modulestore @@ -217,6 +224,42 @@ def _using_temp_index(status_cb: Callable[[str], None] | None = None) -> Generat _wait_for_meili_task(client.delete_index(temp_index_name)) +def _index_is_empty(index_name: str) -> bool: + """ + Check if an index is empty + + Args: + index_name (str): The name of the index to check + """ + client = _get_meilisearch_client() + index = client.get_index(index_name) + return index.get_stats().number_of_documents == 0 + + +def _configure_index(index_name): + """ + Configure the index. The following index settings are best changed on an empty index. + Changing them on a populated index will "re-index all documents in the index", which can take some time. + + Args: + index_name (str): The name of the index to configure + """ + client = _get_meilisearch_client() + + # Mark usage_key as unique (it's not the primary key for the index, but nevertheless must be unique): + client.index(index_name).update_distinct_attribute(INDEX_DISTINCT_ATTRIBUTE) + # Mark which attributes can be used for filtering/faceted search: + client.index(index_name).update_filterable_attributes(INDEX_FILTERABLE_ATTRIBUTES) + # Mark which attributes are used for keyword search, in order of importance: + client.index(index_name).update_searchable_attributes(INDEX_SEARCHABLE_ATTRIBUTES) + # Mark which attributes can be used for sorting search results: + client.index(index_name).update_sortable_attributes(INDEX_SORTABLE_ATTRIBUTES) + + # Update the search ranking rules to let the (optional) "sort" parameter take precedence over keyword relevance. + # cf https://www.meilisearch.com/docs/learn/core_concepts/relevancy + client.index(index_name).update_ranking_rules(INDEX_RANKING_RULES) + + def _recurse_children(block, fn, status_cb: Callable[[str], None] | None = None) -> None: """ Recurse the children of an XBlock and call the given function for each @@ -279,8 +322,75 @@ def is_meilisearch_enabled() -> bool: return False -# pylint: disable=too-many-statements -def rebuild_index(status_cb: Callable[[str], None] | None = None) -> None: +def reset_index(status_cb: Callable[[str], None] | None = None) -> None: + """ + Reset the Meilisearch index, deleting all documents and reconfiguring it + """ + if status_cb is None: + status_cb = log.info + + status_cb("Creating new empty index...") + with _using_temp_index(status_cb) as temp_index_name: + _configure_index(temp_index_name) + status_cb("Index recreated!") + status_cb("Index reset complete.") + + +def _is_index_configured(index_name: str) -> bool: + """ + Check if an index is completely configured + + Args: + index_name (str): The name of the index to check + """ + client = _get_meilisearch_client() + index = client.get_index(index_name) + index_settings = index.get_settings() + for k, v in ( + ("distinctAttribute", INDEX_DISTINCT_ATTRIBUTE), + ("filterableAttributes", INDEX_FILTERABLE_ATTRIBUTES), + ("searchableAttributes", INDEX_SEARCHABLE_ATTRIBUTES), + ("sortableAttributes", INDEX_SORTABLE_ATTRIBUTES), + ("rankingRules", INDEX_RANKING_RULES), + ): + setting = index_settings.get(k, []) + if isinstance(v, list): + v = set(v) + setting = set(setting) + if setting != v: + return False + return True + + +def init_index(status_cb: Callable[[str], None] | None = None, warn_cb: Callable[[str], None] | None = None) -> None: + """ + Initialize the Meilisearch index, creating it and configuring it if it doesn't exist + """ + if status_cb is None: + status_cb = log.info + if warn_cb is None: + warn_cb = log.warning + + if _index_exists(STUDIO_INDEX_NAME): + if _index_is_empty(STUDIO_INDEX_NAME): + warn_cb( + "The studio search index is empty. Please run ./manage.py cms reindex_studio" + " --experimental [--incremental]" + ) + return + if not _is_index_configured(STUDIO_INDEX_NAME): + warn_cb( + "A rebuild of the index is required. Please run ./manage.py cms reindex_studio" + " --experimental [--incremental]" + ) + return + status_cb("Index already exists and is configured.") + return + + reset_index(status_cb) + + +def rebuild_index(status_cb: Callable[[str], None] | None = None, incremental=False) -> None: # lint-amnesty, pylint: disable=too-many-statements """ Rebuild the Meilisearch index from scratch """ @@ -292,7 +402,14 @@ def rebuild_index(status_cb: Callable[[str], None] | None = None) -> None: # Get the lists of libraries status_cb("Counting libraries...") - lib_keys = [lib.library_key for lib in lib_api.ContentLibrary.objects.select_related('org').only('org', 'slug')] + keys_indexed = [] + if incremental: + keys_indexed = list(IncrementalIndexCompleted.objects.values_list("context_key", flat=True)) + lib_keys = [ + lib.library_key + for lib in lib_api.ContentLibrary.objects.select_related("org").only("org", "slug").order_by("-id") + if lib.library_key not in keys_indexed + ] num_libraries = len(lib_keys) # Get the list of courses @@ -300,88 +417,25 @@ def rebuild_index(status_cb: Callable[[str], None] | None = None) -> None: num_courses = CourseOverview.objects.count() # Some counters so we can track our progress as indexing progresses: - num_contexts = num_courses + num_libraries - num_contexts_done = 0 # How many courses/libraries we've indexed + num_libs_skipped = len(keys_indexed) + num_contexts = num_courses + num_libraries + num_libs_skipped + num_contexts_done = 0 + num_libs_skipped # How many courses/libraries we've indexed num_blocks_done = 0 # How many individual components/XBlocks we've indexed status_cb(f"Found {num_courses} courses, {num_libraries} libraries.") - with _using_temp_index(status_cb) as temp_index_name: + with _using_temp_index(status_cb) if not incremental else nullcontext(STUDIO_INDEX_NAME) as index_name: ############## Configure the index ############## - # The following index settings are best changed on an empty index. - # Changing them on a populated index will "re-index all documents in the index, which can take some time" + # The index settings are best changed on an empty index. + # Changing them on a populated index will "re-index all documents in the index", which can take some time # and use more RAM. Instead, we configure an empty index then populate it one course/library at a time. - - # Mark usage_key as unique (it's not the primary key for the index, but nevertheless must be unique): - client.index(temp_index_name).update_distinct_attribute(Fields.usage_key) - # Mark which attributes can be used for filtering/faceted search: - client.index(temp_index_name).update_filterable_attributes([ - # Get specific block/collection using combination of block_id and context_key - Fields.block_id, - Fields.block_type, - Fields.context_key, - Fields.usage_key, - Fields.org, - Fields.tags, - Fields.tags + "." + Fields.tags_taxonomy, - Fields.tags + "." + Fields.tags_level0, - Fields.tags + "." + Fields.tags_level1, - Fields.tags + "." + Fields.tags_level2, - Fields.tags + "." + Fields.tags_level3, - Fields.collections, - Fields.collections + "." + Fields.collections_display_name, - Fields.collections + "." + Fields.collections_key, - Fields.type, - Fields.access_id, - Fields.last_published, - Fields.content + "." + Fields.problem_types, - ]) - # Mark which attributes are used for keyword search, in order of importance: - client.index(temp_index_name).update_searchable_attributes([ - # Keyword search does _not_ search the course name, course ID, breadcrumbs, block type, or other fields. - Fields.display_name, - Fields.block_id, - Fields.content, - Fields.description, - Fields.tags, - Fields.collections, - # If we don't list the following sub-fields _explicitly_, they're only sometimes searchable - that is, they - # are searchable only if at least one document in the index has a value. If we didn't list them here and, - # say, there were no tags.level3 tags in the index, the client would get an error if trying to search for - # these sub-fields: "Attribute `tags.level3` is not searchable." - Fields.tags + "." + Fields.tags_taxonomy, - Fields.tags + "." + Fields.tags_level0, - Fields.tags + "." + Fields.tags_level1, - Fields.tags + "." + Fields.tags_level2, - Fields.tags + "." + Fields.tags_level3, - Fields.collections + "." + Fields.collections_display_name, - Fields.collections + "." + Fields.collections_key, - Fields.published + "." + Fields.display_name, - Fields.published + "." + Fields.published_description, - ]) - # Mark which attributes can be used for sorting search results: - client.index(temp_index_name).update_sortable_attributes([ - Fields.display_name, - Fields.created, - Fields.modified, - Fields.last_published, - ]) - - # Update the search ranking rules to let the (optional) "sort" parameter take precedence over keyword relevance. - # cf https://www.meilisearch.com/docs/learn/core_concepts/relevancy - client.index(temp_index_name).update_ranking_rules([ - "sort", - "words", - "typo", - "proximity", - "attribute", - "exactness", - ]) + if not incremental: + _configure_index(index_name) ############## Libraries ############## status_cb("Indexing libraries...") - def index_library(lib_key: str) -> list: + def index_library(lib_key: LibraryLocatorV2) -> list: docs = [] for component in lib_api.get_library_components(lib_key): try: @@ -396,7 +450,7 @@ def index_library(lib_key: str) -> list: if docs: try: # Add all the docs in this library at once (usually faster than adding one at a time): - _wait_for_meili_task(client.index(temp_index_name).add_documents(docs)) + _wait_for_meili_task(client.index(index_name).add_documents(docs)) except (TypeError, KeyError, MeilisearchError) as err: status_cb(f"Error indexing library {lib_key}: {err}") return docs @@ -416,7 +470,7 @@ def index_collection_batch(batch, num_done, library_key) -> int: if docs: try: # Add docs in batch of 100 at once (usually faster than adding one at a time): - _wait_for_meili_task(client.index(temp_index_name).add_documents(docs)) + _wait_for_meili_task(client.index(index_name).add_documents(docs)) except (TypeError, KeyError, MeilisearchError) as err: status_cb(f"Error indexing collection batch {p}: {err}") return num_done @@ -439,6 +493,8 @@ def index_collection_batch(batch, num_done, library_key) -> int: num_collections_done, lib_key, ) + if incremental: + IncrementalIndexCompleted.objects.get_or_create(context_key=lib_key) status_cb(f"{num_collections_done}/{num_collections} collections indexed for library {lib_key}") num_contexts_done += 1 @@ -464,7 +520,7 @@ def add_with_children(block): if docs: # Add all the docs in this course at once (usually faster than adding one at a time): - _wait_for_meili_task(client.index(temp_index_name).add_documents(docs)) + _wait_for_meili_task(client.index(index_name).add_documents(docs)) return docs paginator = Paginator(CourseOverview.objects.only('id', 'display_name'), 1000) @@ -473,10 +529,16 @@ def add_with_children(block): status_cb( f"{num_contexts_done + 1}/{num_contexts}. Now indexing course {course.display_name} ({course.id})" ) + if course.id in keys_indexed: + num_contexts_done += 1 + continue course_docs = index_course(course) + if incremental: + IncrementalIndexCompleted.objects.get_or_create(context_key=course.id) num_contexts_done += 1 num_blocks_done += len(course_docs) + IncrementalIndexCompleted.objects.all().delete() status_cb(f"Done! {num_blocks_done} blocks indexed across {num_contexts_done} courses, collections and libraries.") diff --git a/openedx/core/djangoapps/content/search/index_config.py b/openedx/core/djangoapps/content/search/index_config.py new file mode 100644 index 000000000000..9570956e425e --- /dev/null +++ b/openedx/core/djangoapps/content/search/index_config.py @@ -0,0 +1,70 @@ +"""Configuration for the search index.""" +from .documents import Fields + + +INDEX_DISTINCT_ATTRIBUTE = "usage_key" + +# Mark which attributes can be used for filtering/faceted search: +INDEX_FILTERABLE_ATTRIBUTES = [ + # Get specific block/collection using combination of block_id and context_key + Fields.block_id, + Fields.block_type, + Fields.context_key, + Fields.usage_key, + Fields.org, + Fields.tags, + Fields.tags + "." + Fields.tags_taxonomy, + Fields.tags + "." + Fields.tags_level0, + Fields.tags + "." + Fields.tags_level1, + Fields.tags + "." + Fields.tags_level2, + Fields.tags + "." + Fields.tags_level3, + Fields.collections, + Fields.collections + "." + Fields.collections_display_name, + Fields.collections + "." + Fields.collections_key, + Fields.type, + Fields.access_id, + Fields.last_published, + Fields.content + "." + Fields.problem_types, +] + +# Mark which attributes are used for keyword search, in order of importance: +INDEX_SEARCHABLE_ATTRIBUTES = [ + # Keyword search does _not_ search the course name, course ID, breadcrumbs, block type, or other fields. + Fields.display_name, + Fields.block_id, + Fields.content, + Fields.description, + Fields.tags, + Fields.collections, + # If we don't list the following sub-fields _explicitly_, they're only sometimes searchable - that is, they + # are searchable only if at least one document in the index has a value. If we didn't list them here and, + # say, there were no tags.level3 tags in the index, the client would get an error if trying to search for + # these sub-fields: "Attribute `tags.level3` is not searchable." + Fields.tags + "." + Fields.tags_taxonomy, + Fields.tags + "." + Fields.tags_level0, + Fields.tags + "." + Fields.tags_level1, + Fields.tags + "." + Fields.tags_level2, + Fields.tags + "." + Fields.tags_level3, + Fields.collections + "." + Fields.collections_display_name, + Fields.collections + "." + Fields.collections_key, + Fields.published + "." + Fields.display_name, + Fields.published + "." + Fields.published_description, +] + +# Mark which attributes can be used for sorting search results: +INDEX_SORTABLE_ATTRIBUTES = [ + Fields.display_name, + Fields.created, + Fields.modified, + Fields.last_published, +] + +# Update the search ranking rules to let the (optional) "sort" parameter take precedence over keyword relevance. +INDEX_RANKING_RULES = [ + "sort", + "words", + "typo", + "proximity", + "attribute", + "exactness", +] diff --git a/openedx/core/djangoapps/content/search/management/commands/reindex_studio.py b/openedx/core/djangoapps/content/search/management/commands/reindex_studio.py index 3767ebcba6c9..2d8bb29f7a1f 100644 --- a/openedx/core/djangoapps/content/search/management/commands/reindex_studio.py +++ b/openedx/core/djangoapps/content/search/management/commands/reindex_studio.py @@ -18,8 +18,11 @@ class Command(BaseCommand): """ def add_arguments(self, parser): - parser.add_argument('--experimental', action='store_true') - parser.set_defaults(experimental=False) + parser.add_argument("--experimental", action="store_true") + parser.add_argument("--reset", action="store_true") + parser.add_argument("--init", action="store_true") + parser.add_argument("--incremental", action="store_true") + parser.set_defaults(experimental=False, reset=False, init=False, incremental=False) def handle(self, *args, **options): """ @@ -34,4 +37,11 @@ def handle(self, *args, **options): "Use the --experimental argument to acknowledge and run it." ) - api.rebuild_index(self.stdout.write) + if options["reset"]: + api.reset_index(self.stdout.write) + elif options["init"]: + api.init_index(self.stdout.write, self.stderr.write) + elif options["incremental"]: + api.rebuild_index(self.stdout.write, incremental=True) + else: + api.rebuild_index(self.stdout.write) diff --git a/openedx/core/djangoapps/content/search/migrations/0002_incrementalindexcompleted.py b/openedx/core/djangoapps/content/search/migrations/0002_incrementalindexcompleted.py new file mode 100644 index 000000000000..a316c35a7dfe --- /dev/null +++ b/openedx/core/djangoapps/content/search/migrations/0002_incrementalindexcompleted.py @@ -0,0 +1,21 @@ +# Generated by Django 4.2.16 on 2024-11-15 12:40 + +from django.db import migrations, models +import opaque_keys.edx.django.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('search', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='IncrementalIndexCompleted', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('context_key', opaque_keys.edx.django.models.LearningContextKeyField(max_length=255, unique=True)), + ], + ), + ] diff --git a/openedx/core/djangoapps/content/search/models.py b/openedx/core/djangoapps/content/search/models.py index 711c493ff895..6fa53ef17b34 100644 --- a/openedx/core/djangoapps/content/search/models.py +++ b/openedx/core/djangoapps/content/search/models.py @@ -65,3 +65,15 @@ def get_access_ids_for_request(request: Request, omit_orgs: list[str] = None) -> course_clause | library_clause ).order_by('-id').values_list("id", flat=True) ) + + +class IncrementalIndexCompleted(models.Model): + """ + Stores the contex keys of aleady indexed courses and libraries for incremental indexing. + """ + + context_key = LearningContextKeyField( + max_length=255, + unique=True, + null=False, + ) diff --git a/openedx/core/djangoapps/content/search/tests/test_api.py b/openedx/core/djangoapps/content/search/tests/test_api.py index 0aa762fd187f..c9c2b2589a31 100644 --- a/openedx/core/djangoapps/content/search/tests/test_api.py +++ b/openedx/core/djangoapps/content/search/tests/test_api.py @@ -10,8 +10,10 @@ from opaque_keys.edx.keys import UsageKey import ddt +import pytest from django.test import override_settings from freezegun import freeze_time +from meilisearch.errors import MeilisearchApiError from openedx_learning.api import authoring as authoring_api from organizations.tests.factories import OrganizationFactory @@ -26,7 +28,7 @@ try: # This import errors in the lms because content.search is not an installed app there. from .. import api - from ..models import SearchAccess + from ..models import SearchAccess, IncrementalIndexCompleted except RuntimeError: SearchAccess = {} @@ -239,6 +241,87 @@ def test_reindex_meilisearch(self, mock_meilisearch): any_order=True, ) + @override_settings(MEILISEARCH_ENABLED=True) + def test_reindex_meilisearch_incremental(self, mock_meilisearch): + + # Add tags field to doc, since reindex calls includes tags + doc_sequential = copy.deepcopy(self.doc_sequential) + doc_sequential["tags"] = {} + doc_vertical = copy.deepcopy(self.doc_vertical) + doc_vertical["tags"] = {} + doc_problem1 = copy.deepcopy(self.doc_problem1) + doc_problem1["tags"] = {} + doc_problem1["collections"] = {"display_name": [], "key": []} + doc_problem2 = copy.deepcopy(self.doc_problem2) + doc_problem2["tags"] = {} + doc_problem2["collections"] = {"display_name": [], "key": []} + doc_collection = copy.deepcopy(self.collection_dict) + doc_collection["tags"] = {} + + api.rebuild_index(incremental=True) + assert mock_meilisearch.return_value.index.return_value.add_documents.call_count == 3 + mock_meilisearch.return_value.index.return_value.add_documents.assert_has_calls( + [ + call([doc_sequential, doc_vertical]), + call([doc_problem1, doc_problem2]), + call([doc_collection]), + ], + any_order=True, + ) + + # Now we simulate interruption by passing this function to the status_cb argument + def simulated_interruption(message): + # this exception prevents courses from being indexed + if "Indexing courses" in message: + raise Exception("Simulated interruption") + + with pytest.raises(Exception, match="Simulated interruption"): + api.rebuild_index(simulated_interruption, incremental=True) + + # two more calls due to collections + assert mock_meilisearch.return_value.index.return_value.add_documents.call_count == 5 + assert IncrementalIndexCompleted.objects.all().count() == 1 + api.rebuild_index(incremental=True) + assert IncrementalIndexCompleted.objects.all().count() == 0 + # one missing course indexed + assert mock_meilisearch.return_value.index.return_value.add_documents.call_count == 6 + + @override_settings(MEILISEARCH_ENABLED=True) + def test_reset_meilisearch_index(self, mock_meilisearch): + api.reset_index() + mock_meilisearch.return_value.swap_indexes.assert_called_once() + mock_meilisearch.return_value.create_index.assert_called_once() + mock_meilisearch.return_value.delete_index.call_count = 2 + api.reset_index() + mock_meilisearch.return_value.delete_index.call_count = 4 + + @override_settings(MEILISEARCH_ENABLED=True) + def test_init_meilisearch_index(self, mock_meilisearch): + # Test index already exists + api.init_index() + mock_meilisearch.return_value.swap_indexes.assert_not_called() + mock_meilisearch.return_value.create_index.assert_not_called() + mock_meilisearch.return_value.delete_index.assert_not_called() + + # Test index already exists and has no documents + mock_meilisearch.return_value.get_stats.return_value = 0 + api.init_index() + mock_meilisearch.return_value.swap_indexes.assert_not_called() + mock_meilisearch.return_value.create_index.assert_not_called() + mock_meilisearch.return_value.delete_index.assert_not_called() + + mock_meilisearch.return_value.get_index.side_effect = [ + MeilisearchApiError("Testing reindex", Mock(text='{"code":"index_not_found"}')), + MeilisearchApiError("Testing reindex", Mock(text='{"code":"index_not_found"}')), + Mock(created_at=1), + Mock(created_at=1), + Mock(created_at=1), + ] + api.init_index() + mock_meilisearch.return_value.swap_indexes.assert_called_once() + mock_meilisearch.return_value.create_index.assert_called_once() + mock_meilisearch.return_value.delete_index.call_count = 2 + @override_settings(MEILISEARCH_ENABLED=True) @patch( "openedx.core.djangoapps.content.search.api.searchable_doc_for_collection", diff --git a/openedx/core/djangoapps/content_libraries/api.py b/openedx/core/djangoapps/content_libraries/api.py index 85cd2f2c06e8..5195826468c2 100644 --- a/openedx/core/djangoapps/content_libraries/api.py +++ b/openedx/core/djangoapps/content_libraries/api.py @@ -308,6 +308,13 @@ class LibraryXBlockType: # ============ +def user_can_create_library(user: AbstractUser) -> bool: + """ + Check if the user has permission to create a content library. + """ + return user.has_perm(permissions.CAN_CREATE_CONTENT_LIBRARY) + + def get_libraries_for_user(user, org=None, text_search=None, order=None): """ Return content libraries that the user has permission to view. diff --git a/openedx/core/djangoapps/content_libraries/permissions.py b/openedx/core/djangoapps/content_libraries/permissions.py index 17671b5659f8..4e72381986ed 100644 --- a/openedx/core/djangoapps/content_libraries/permissions.py +++ b/openedx/core/djangoapps/content_libraries/permissions.py @@ -48,6 +48,12 @@ def is_studio_request(_): return settings.SERVICE_VARIANT == "cms" +@blanket_rule +def is_course_creator(user): + from cms.djangoapps.course_creators.views import get_course_creator_status + + return get_course_creator_status(user) == 'granted' + ########################### Permissions ########################### # Is the user allowed to view XBlocks from the specified content library @@ -68,7 +74,10 @@ def is_studio_request(_): # Is the user allowed to create content libraries? CAN_CREATE_CONTENT_LIBRARY = 'content_libraries.create_library' -perms[CAN_CREATE_CONTENT_LIBRARY] = is_user_active +if settings.FEATURES.get('ENABLE_CREATOR_GROUP', False): + perms[CAN_CREATE_CONTENT_LIBRARY] = is_global_staff | (is_user_active & is_course_creator) +else: + perms[CAN_CREATE_CONTENT_LIBRARY] = is_global_staff # Is the user allowed to view the specified content library in Studio, # including to view the raw OLX and asset files? @@ -76,8 +85,8 @@ def is_studio_request(_): perms[CAN_VIEW_THIS_CONTENT_LIBRARY] = is_user_active & ( # Global staff can access any library is_global_staff | - # Some libraries allow anyone to view them in Studio: - Attribute('allow_public_read', True) | + # Libraries with "public read" permissions can be accessed only by course creators + (Attribute('allow_public_read', True) & is_course_creator) | # Otherwise the user must be part of the library's team has_explicit_read_permission_for_library ) diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index 638c053f62c3..77de1c028aa7 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -71,7 +71,7 @@ class ContentLibrariesRestApiTest(APITransactionTestCase): def setUp(self): super().setUp() - self.user = UserFactory.create(username="Bob", email="bob@example.com", password="edx") + self.user = UserFactory.create(username="Bob", email="bob@example.com", password="edx", is_staff=True) # Create an organization self.organization, _ = Organization.objects.get_or_create( short_name="CL-TEST", diff --git a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py index 18d4ec591604..83b277604071 100644 --- a/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py +++ b/openedx/core/djangoapps/content_libraries/tests/test_content_libraries.py @@ -259,6 +259,8 @@ def test_library_blocks(self): # pylint: disable=too-many-statements Tests with some non-ASCII chars in slugs, titles, descriptions. """ + admin = UserFactory.create(username="Admin", email="admin@example.com", is_staff=True) + lib = self._create_library(slug="téstlꜟط", title="A Tést Lꜟطrary", description="Tésting XBlocks") lib_id = lib["id"] assert lib['has_unpublished_changes'] is False @@ -531,7 +533,7 @@ def test_library_permissions(self): # pylint: disable=too-many-statements Learning Core data models. """ # Create a few users to use for all of these tests: - admin = UserFactory.create(username="Admin", email="admin@example.com") + admin = UserFactory.create(username="Admin", email="admin@example.com", is_staff=True) author = UserFactory.create(username="Author", email="author@example.com") reader = UserFactory.create(username="Reader", email="reader@example.com") group = Group.objects.create(name="group1") @@ -653,14 +655,15 @@ def test_library_permissions(self): # pylint: disable=too-many-statements self._get_library_block_asset(block3_key, file_name="static/whatever.png", expect_response=403) # Nor can they preview the block: self._render_block_view(block3_key, view_name="student_view", expect_response=403) - # But if we grant allow_public_read, then they can: + # Even if we grant allow_public_read, then they can't: with self.as_user(admin): self._update_library(lib_id, allow_public_read=True) self._set_library_block_asset(block3_key, "static/whatever.png", b"data") with self.as_user(random_user): - self._get_library_block_olx(block3_key) + self._get_library_block_olx(block3_key, expect_response=403) + self._get_library_block_fields(block3_key, expect_response=403) + # But he can preview the block: self._render_block_view(block3_key, view_name="student_view") - f = self._get_library_block_fields(block3_key) # self._get_library_block_assets(block3_key) # self._get_library_block_asset(block3_key, file_name="whatever.png") @@ -702,7 +705,7 @@ def test_no_lockout(self): """ Test that administrators cannot be removed if they are the only administrator granted access. """ - admin = UserFactory.create(username="Admin", email="admin@example.com") + admin = UserFactory.create(username="Admin", email="admin@example.com", is_staff=True) successor = UserFactory.create(username="Successor", email="successor@example.com") with self.as_user(admin): lib = self._create_library(slug="permtest", title="Permission Test Library", description="Testing") @@ -1026,7 +1029,7 @@ def test_library_paste_clipboard(self): from openedx.core.djangoapps.content_staging.api import save_xblock_to_user_clipboard # Create user to perform tests on - author = UserFactory.create(username="Author", email="author@example.com") + author = UserFactory.create(username="Author", email="author@example.com", is_staff=True) with self.as_user(author): lib = self._create_library( slug="test_lib_paste_clipboard", diff --git a/openedx/core/djangoapps/content_libraries/views.py b/openedx/core/djangoapps/content_libraries/views.py index e30e58e75a26..3dc7f538df86 100644 --- a/openedx/core/djangoapps/content_libraries/views.py +++ b/openedx/core/djangoapps/content_libraries/views.py @@ -710,6 +710,9 @@ class LibraryBlockOlxView(APIView): @convert_exceptions def get(self, request, usage_key_str): """ + DEPRECATED. Use get_block_olx_view() in xblock REST-API. + Can be removed post-Teak. + Get the block's OLX """ key = LibraryUsageLocatorV2.from_string(usage_key_str) diff --git a/openedx/core/djangoapps/content_tagging/rest_api/v1/tests/test_views.py b/openedx/core/djangoapps/content_tagging/rest_api/v1/tests/test_views.py index 463ea42d08e9..5a99868ece66 100644 --- a/openedx/core/djangoapps/content_tagging/rest_api/v1/tests/test_views.py +++ b/openedx/core/djangoapps/content_tagging/rest_api/v1/tests/test_views.py @@ -13,6 +13,7 @@ import ddt from django.contrib.auth import get_user_model from django.core.files.uploadedfile import SimpleUploadedFile +from edx_django_utils.cache import RequestCache from opaque_keys.edx.locator import BlockUsageLocator, CourseLocator, LibraryCollectionLocator from openedx_tagging.core.tagging.models import Tag, Taxonomy from openedx_tagging.core.tagging.models.system_defined import SystemDefinedTaxonomy @@ -34,7 +35,6 @@ from openedx.core.djangoapps.content_libraries.api import AccessLevel, create_library, set_library_user_permissions from openedx.core.djangoapps.content_tagging import api as tagging_api from openedx.core.djangoapps.content_tagging.models import TaxonomyOrg -from openedx.core.djangoapps.content_tagging.utils import rules_cache from openedx.core.djangolib.testing.utils import skip_unless_cms from ....tests.test_objecttag_export_helpers import TaggedCourseMixin @@ -289,8 +289,8 @@ def setUp(self): self._setUp_taxonomies() self._setUp_collection() - # Clear the rules cache in between test runs to keep query counts consistent. - rules_cache.clear() + # Clear all request caches in between test runs to keep query counts consistent. + RequestCache.clear_all_namespaces() @skip_unless_cms @@ -510,12 +510,12 @@ def test_create_taxonomy(self, user_attr: str, expected_status: int) -> None: @ddt.data( ('staff', 11), - ("content_creatorA", 16), - ("library_staffA", 16), - ("library_userA", 16), - ("instructorA", 16), - ("course_instructorA", 16), - ("course_staffA", 16), + ("content_creatorA", 17), + ("library_staffA", 17), + ("library_userA", 17), + ("instructorA", 17), + ("course_instructorA", 17), + ("course_staffA", 17), ) @ddt.unpack def test_list_taxonomy_query_count(self, user_attr: str, expected_queries: int): @@ -1879,16 +1879,16 @@ def test_get_copied_tags(self): ('staff', 'courseA', 8), ('staff', 'libraryA', 8), ('staff', 'collection_key', 8), - ("content_creatorA", 'courseA', 11, False), - ("content_creatorA", 'libraryA', 11, False), - ("content_creatorA", 'collection_key', 11, False), - ("library_staffA", 'libraryA', 11, False), # Library users can only view objecttags, not change them? - ("library_staffA", 'collection_key', 11, False), - ("library_userA", 'libraryA', 11, False), - ("library_userA", 'collection_key', 11, False), - ("instructorA", 'courseA', 11), - ("course_instructorA", 'courseA', 11), - ("course_staffA", 'courseA', 11), + ("content_creatorA", 'courseA', 12, False), + ("content_creatorA", 'libraryA', 12, False), + ("content_creatorA", 'collection_key', 12, False), + ("library_staffA", 'libraryA', 12, False), # Library users can only view objecttags, not change them? + ("library_staffA", 'collection_key', 12, False), + ("library_userA", 'libraryA', 12, False), + ("library_userA", 'collection_key', 12, False), + ("instructorA", 'courseA', 12), + ("course_instructorA", 'courseA', 12), + ("course_staffA", 'courseA', 12), ) @ddt.unpack def test_object_tags_query_count( diff --git a/openedx/core/djangoapps/discussions/config/waffle.py b/openedx/core/djangoapps/discussions/config/waffle.py index 7b6395bb1c01..05fc24eeb3b7 100644 --- a/openedx/core/djangoapps/discussions/config/waffle.py +++ b/openedx/core/djangoapps/discussions/config/waffle.py @@ -2,6 +2,7 @@ This module contains various configuration settings via waffle switches for the discussions app. """ +from django.conf import settings from openedx.core.djangoapps.waffle_utils import CourseWaffleFlag @@ -57,6 +58,18 @@ def is_forum_v2_enabled(course_id): """ - Returns a boolean if forum V2 is enabled on the course + Returns whether forum V2 is enabled on the course. This is a 2-step check: + + 1. Check value of settings.DISABLE_FORUM_V2: if it exists and is true, this setting overrides any course flag. + 2. Else, check the value of the corresponding course waffle flag. """ + if is_forum_v2_disabled_globally(): + return False return ENABLE_FORUM_V2.is_enabled(course_id) + + +def is_forum_v2_disabled_globally() -> bool: + """ + Return True if DISABLE_FORUM_V2 is defined and true-ish. + """ + return getattr(settings, "DISABLE_FORUM_V2", False) diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/models.py b/openedx/core/djangoapps/django_comment_common/comment_client/models.py index 31256eb64735..094475c81fb5 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/models.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/models.py @@ -2,10 +2,11 @@ import logging +import typing as t from .utils import CommentClientRequestError, extract, perform_request, get_course_key from forum import api as forum_api -from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled, is_forum_v2_disabled_globally log = logging.getLogger(__name__) @@ -72,13 +73,14 @@ def retrieve(self, *args, **kwargs): def _retrieve(self, *args, **kwargs): course_id = self.attributes.get("course_id") or kwargs.get("course_id") - if not course_id: - course_id = forum_api.get_course_id_by_comment(self.id) - course_key = get_course_key(course_id) + if course_id: + use_forumv2 = is_forum_v2_enabled(course_id) + else: + use_forumv2, course_id = is_forum_v2_enabled_for_comment(self.id) response = None - if is_forum_v2_enabled(course_key): + if use_forumv2: if self.type == "comment": - response = forum_api.get_parent_comment(comment_id=self.attributes["id"], course_id=str(course_key)) + response = forum_api.get_parent_comment(comment_id=self.attributes["id"], course_id=course_id) if response is None: raise CommentClientRequestError("Forum v2 API call is missing") else: @@ -369,3 +371,22 @@ def handle_create_thread(self, course_id): context=request_data.get("context", None), ) return response + + +def is_forum_v2_enabled_for_comment(comment_id: str) -> tuple[bool, t.Optional[str]]: + """ + Figure out whether we use forum v2 for a given comment. + + See is_forum_v2_enabled_for_thread. + + Return: + + enabled (bool) + course_id (str or None) + """ + if is_forum_v2_disabled_globally(): + return False, None + + course_id = forum_api.get_course_id_by_comment(comment_id) + course_key = get_course_key(course_id) + return is_forum_v2_enabled(course_key), course_id diff --git a/openedx/core/djangoapps/django_comment_common/comment_client/thread.py b/openedx/core/djangoapps/django_comment_common/comment_client/thread.py index 74aa8358f112..b1f9dbd08d7e 100644 --- a/openedx/core/djangoapps/django_comment_common/comment_client/thread.py +++ b/openedx/core/djangoapps/django_comment_common/comment_client/thread.py @@ -2,12 +2,13 @@ import logging +import typing as t from eventtracking import tracker from . import models, settings, utils from forum import api as forum_api -from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled +from openedx.core.djangoapps.discussions.config.waffle import is_forum_v2_enabled, is_forum_v2_disabled_globally log = logging.getLogger(__name__) @@ -172,16 +173,17 @@ def _retrieve(self, *args, **kwargs): } request_params = utils.strip_none(request_params) course_id = kwargs.get("course_id") - if not course_id: - course_id = forum_api.get_course_id_by_thread(self.id) - course_key = utils.get_course_key(course_id) - if is_forum_v2_enabled(course_key): + if course_id: + use_forumv2 = is_forum_v2_enabled(course_id) + else: + use_forumv2, course_id = is_forum_v2_enabled_for_thread(self.id) + if use_forumv2: if user_id := request_params.get('user_id'): request_params['user_id'] = str(user_id) response = forum_api.get_thread( thread_id=self.id, params=request_params, - course_id=str(course_key) + course_id=course_id, ) else: response = utils.perform_request( @@ -296,3 +298,28 @@ def _url_for_pin_thread(thread_id): def _url_for_un_pin_thread(thread_id): return f"{settings.PREFIX}/threads/{thread_id}/unpin" + + +def is_forum_v2_enabled_for_thread(thread_id: str) -> tuple[bool, t.Optional[str]]: + """ + Figure out whether we use forum v2 for a given thread. + + This is a complex affair... First, we check the value of the DISABLE_FORUM_V2 + setting, which overrides everything. If this setting does not exist, then we need to + find the course ID that corresponds to the thread ID. Then, we return the value of + the course waffle flag for this course ID. + + Note that to fetch the course ID associated to a thread ID, we need to connect both + to mongodb and mysql. As a consequence, when forum v2 needs adequate connection + strings for both backends. + + Return: + + enabled (bool) + course_id (str or None) + """ + if is_forum_v2_disabled_globally(): + return False, None + course_id = forum_api.get_course_id_by_thread(thread_id) + course_key = utils.get_course_key(course_id) + return is_forum_v2_enabled(course_key), course_id diff --git a/openedx/core/djangoapps/xblock/api.py b/openedx/core/djangoapps/xblock/api.py index 00bb8bc356a6..35c05cd7b407 100644 --- a/openedx/core/djangoapps/xblock/api.py +++ b/openedx/core/djangoapps/xblock/api.py @@ -33,7 +33,12 @@ LearningCoreXBlockRuntime, ) from .data import CheckPerm, LatestVersion -from .utils import get_secure_token_for_xblock_handler, get_xblock_id_for_anonymous_user +from .rest_api.url_converters import VersionConverter +from .utils import ( + get_secure_token_for_xblock_handler, + get_xblock_id_for_anonymous_user, + get_auto_latest_version, +) from .runtime.learning_core_runtime import LearningCoreXBlockRuntime @@ -208,13 +213,26 @@ def get_component_from_usage_key(usage_key: UsageKeyV2) -> Component: ) -def get_block_draft_olx(usage_key: UsageKeyV2) -> str: +def get_block_olx( + usage_key: UsageKeyV2, + *, + version: int | LatestVersion = LatestVersion.AUTO +) -> str: """ - Get the OLX source of the draft version of the given Learning-Core-backed XBlock. + Get the OLX source of the of the given Learning-Core-backed XBlock and a version. """ - # Inefficient but simple approach. Optimize later if needed. component = get_component_from_usage_key(usage_key) - component_version = component.versioning.draft + version = get_auto_latest_version(version) + + if version == LatestVersion.DRAFT: + component_version = component.versioning.draft + elif version == LatestVersion.PUBLISHED: + component_version = component.versioning.published + else: + assert isinstance(version, int) + component_version = component.versioning.version_num(version) + if component_version is None: + raise NoSuchUsage(usage_key) # TODO: we should probably make a method on ComponentVersion that returns # a content based on the name. Accessing by componentversioncontent__key is @@ -224,6 +242,11 @@ def get_block_draft_olx(usage_key: UsageKeyV2) -> str: return content.text +def get_block_draft_olx(usage_key: UsageKeyV2) -> str: + """ DEPRECATED. Use get_block_olx(). Can be removed post-Teak. """ + return get_block_olx(usage_key, version=LatestVersion.DRAFT) + + def render_block_view(block, view_name, user): # pylint: disable=unused-argument """ Get the HTML, JS, and CSS needed to render the given XBlock view. diff --git a/openedx/core/djangoapps/xblock/rest_api/serializers.py b/openedx/core/djangoapps/xblock/rest_api/serializers.py new file mode 100644 index 000000000000..bb4dd1da229f --- /dev/null +++ b/openedx/core/djangoapps/xblock/rest_api/serializers.py @@ -0,0 +1,11 @@ +""" +Serializers for the xblock REST API +""" +from rest_framework import serializers + + +class XBlockOlxSerializer(serializers.Serializer): + """ + Serializer for representing an XBlock's OLX + """ + olx = serializers.CharField() diff --git a/openedx/core/djangoapps/xblock/rest_api/urls.py b/openedx/core/djangoapps/xblock/rest_api/urls.py index ee41b43f5b39..a83d5104e570 100644 --- a/openedx/core/djangoapps/xblock/rest_api/urls.py +++ b/openedx/core/djangoapps/xblock/rest_api/urls.py @@ -19,6 +19,8 @@ path('', views.block_metadata), # get/post full json fields of an XBlock: path('fields/', views.BlockFieldsView.as_view()), + # Get the OLX source code of the specified block + path('olx/', views.get_block_olx_view), # render one of this XBlock's views (e.g. student_view) path('view//', views.render_block_view), # get the URL needed to call this XBlock's handlers diff --git a/openedx/core/djangoapps/xblock/rest_api/views.py b/openedx/core/djangoapps/xblock/rest_api/views.py index d69edcbfd51c..edcbf22e0d3d 100644 --- a/openedx/core/djangoapps/xblock/rest_api/views.py +++ b/openedx/core/djangoapps/xblock/rest_api/views.py @@ -33,9 +33,11 @@ get_handler_url as _get_handler_url, load_block, render_block_view as _render_block_view, + get_block_olx, ) from ..utils import validate_secure_token_for_xblock_handler from .url_converters import VersionConverter +from .serializers import XBlockOlxSerializer User = get_user_model() @@ -213,6 +215,23 @@ def xblock_handler( return response +@api_view(['GET']) +@view_auth_classes(is_authenticated=False) +def get_block_olx_view( + request, + usage_key: UsageKeyV2, + version: LatestVersion | int = LatestVersion.AUTO, +): + """ + Get the OLX (XML serialization) of the specified XBlock + """ + context_impl = get_learning_context_impl(usage_key) + if not context_impl.can_view_block_for_editing(request.user, usage_key): + raise PermissionDenied(f"You don't have permission to access the OLX of component '{usage_key}'.") + olx = get_block_olx(usage_key, version=version) + return Response(XBlockOlxSerializer({"olx": olx}).data) + + def cors_allow_xblock_handler(sender, request, **kwargs): # lint-amnesty, pylint: disable=unused-argument """ Sandboxed XBlocks need to be able to call XBlock handlers via POST, diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index fd2e867a3a8f..dde2084e54cd 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -24,6 +24,7 @@ from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from openedx.core.lib.xblock_serializer.data import StaticFile from ..data import AuthoredDataMode, LatestVersion +from ..utils import get_auto_latest_version from ..learning_context.manager import get_learning_context_impl from .runtime import XBlockRuntime @@ -178,11 +179,7 @@ def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion # just get it the easy way. component = self._get_component_from_usage_key(usage_key) - if version == LatestVersion.AUTO: - if self.authored_data_mode == AuthoredDataMode.DEFAULT_DRAFT: - version = LatestVersion.DRAFT - else: - version = LatestVersion.PUBLISHED + version = get_auto_latest_version(version) if self.authored_data_mode == AuthoredDataMode.STRICTLY_PUBLISHED and version != LatestVersion.PUBLISHED: raise ValidationError("This runtime only allows accessing the published version of components") if version == LatestVersion.DRAFT: diff --git a/openedx/core/djangoapps/xblock/utils.py b/openedx/core/djangoapps/xblock/utils.py index 375bb9d21450..b4ae054cf498 100644 --- a/openedx/core/djangoapps/xblock/utils.py +++ b/openedx/core/djangoapps/xblock/utils.py @@ -11,6 +11,10 @@ import crum from django.conf import settings +from openedx.core.djangoapps.xblock.apps import get_xblock_app_config + +from .data import AuthoredDataMode, LatestVersion + def get_secure_token_for_xblock_handler(user_id, block_key_str, time_idx=0): """ @@ -167,3 +171,18 @@ def get_xblock_id_for_anonymous_user(user): return current_request.session["xblock_id_for_anonymous_user"] else: raise RuntimeError("Cannot get a user ID for an anonymous user outside of an HTTP request context.") + + +def get_auto_latest_version(version: int | LatestVersion) -> int | LatestVersion: + """ + Gets the actual LatestVersion if is `LatestVersion.AUTO`; + otherwise, returns the same value. + """ + if version == LatestVersion.AUTO: + authored_data_mode = get_xblock_app_config().get_runtime_params()["authored_data_mode"] + version = ( + LatestVersion.DRAFT + if authored_data_mode == AuthoredDataMode.DEFAULT_DRAFT + else LatestVersion.PUBLISHED + ) + return version diff --git a/openedx/features/course_bookmarks/static/course_bookmarks/js/models/bookmark.js b/openedx/features/course_bookmarks/static/course_bookmarks/js/models/bookmark.js index 13671bddf56a..9ad45a5422ce 100644 --- a/openedx/features/course_bookmarks/static/course_bookmarks/js/models/bookmark.js +++ b/openedx/features/course_bookmarks/static/course_bookmarks/js/models/bookmark.js @@ -1,20 +1,37 @@ (function(define) { - 'use strict'; + 'use strict'; - define(['backbone'], function(Backbone) { - return Backbone.Model.extend({ - idAttribute: 'id', - defaults: { - course_id: '', - usage_id: '', - display_name: '', - path: [], - created: '' - }, + define(['backbone'], function(Backbone) { + return Backbone.Model.extend({ + idAttribute: 'id', + defaults: { + course_id: '', + usage_id: '', + display_name: '', + path: [], + created: '' + }, - blockUrl: function() { - return '/courses/' + this.get('course_id') + '/jump_to/' + this.get('usage_id'); - } - }); + blockUrl: function() { + var path = this.get('path'); + var url = '/courses/' + this.get('course_id') + '/jump_to/' + this.get('usage_id'); + var params = new URLSearchParams(); + var usage_id = this.get('usage_id'); + // Confirm that current usage_id does not correspond to current unit + // path contains an array of parent blocks to the bookmarked block. + // Units only have two parents i.e. section and subsections. + // Below condition is only satisfied if a block lower than unit is bookmarked. + if (path.length > 2 && usage_id !== path[path.length - 1]) { + params.append('jumpToId', usage_id); + } + if (params.size > 0) { + // Pass nested block details via query parameters for it to be passed to learning mfe + // The learning mfe should pass it back to unit xblock via iframe url params. + // This would allow us to scroll to the child xblock. + url = url + '?' + params.toString(); + } + return url; + } }); + }); }(define || RequireJS.define)); diff --git a/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmark_button.js b/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmark_button.js index 838f631868dc..3612038842c5 100644 --- a/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmark_button.js +++ b/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmark_button.js @@ -20,6 +20,12 @@ this.bookmarkId = options.bookmarkId; this.bookmarked = options.bookmarked; this.usageId = options.usageId; + if (options.bookmarkedText) { + this.bookmarkedText = options.bookmarkedText; + } + if (options.bookmarkText) { + this.bookmarkText = options.bookmarkText; + } this.setBookmarkState(this.bookmarked); }, diff --git a/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmarks_list.js b/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmarks_list.js index 52f5fbd74c1e..55dd1bd58ae5 100644 --- a/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmarks_list.js +++ b/openedx/features/course_bookmarks/static/course_bookmarks/js/views/bookmarks_list.js @@ -78,9 +78,7 @@ component_type: componentType, component_usage_id: componentUsageId } - ).always(function() { - window.location.href = event.currentTarget.pathname; - }); + ); }, /** diff --git a/package-lock.json b/package-lock.json index c78ef7c76e27..16232c97f71f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,7 +13,7 @@ "@babel/plugin-proposal-object-rest-spread": "^7.18.9", "@babel/plugin-transform-object-assign": "^7.18.6", "@babel/preset-env": "^7.19.0", - "@babel/preset-react": "7.25.9", + "@babel/preset-react": "7.26.3", "@edx/brand-edx.org": "^2.0.7", "@edx/edx-bootstrap": "1.0.4", "@edx/edx-proctoring": "^4.18.1", @@ -1801,9 +1801,9 @@ } }, "node_modules/@babel/preset-react": { - "version": "7.25.9", - "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.25.9.tgz", - "integrity": "sha512-D3to0uSPiWE7rBrdIICCd0tJSIGpLaaGptna2+w7Pft5xMqLpA1sz99DK5TZ1TjGbdQ/VI1eCSZ06dv3lT4JOw==", + "version": "7.26.3", + "resolved": "https://registry.npmjs.org/@babel/preset-react/-/preset-react-7.26.3.tgz", + "integrity": "sha512-Nl03d6T9ky516DGK2YMxrTqvnpUW63TnJMOMonj+Zae0JiPC5BC9xPMSL6L8fiSpA5vP88qfygavVQvnLp+6Cw==", "license": "MIT", "dependencies": { "@babel/helper-plugin-utils": "^7.25.9", @@ -5099,133 +5099,148 @@ "peer": true }, "node_modules/@webassemblyjs/ast": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz", - "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz", + "integrity": "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==", + "license": "MIT", "dependencies": { - "@webassemblyjs/helper-numbers": "1.11.6", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6" + "@webassemblyjs/helper-numbers": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2" } }, "node_modules/@webassemblyjs/floating-point-hex-parser": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz", - "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==" + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.13.2.tgz", + "integrity": "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==", + "license": "MIT" }, "node_modules/@webassemblyjs/helper-api-error": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz", - "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==" + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.13.2.tgz", + "integrity": "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==", + "license": "MIT" }, "node_modules/@webassemblyjs/helper-buffer": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz", - "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==" + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.14.1.tgz", + "integrity": "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==", + "license": "MIT" }, "node_modules/@webassemblyjs/helper-numbers": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz", - "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.13.2.tgz", + "integrity": "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==", + "license": "MIT", "dependencies": { - "@webassemblyjs/floating-point-hex-parser": "1.11.6", - "@webassemblyjs/helper-api-error": "1.11.6", + "@webassemblyjs/floating-point-hex-parser": "1.13.2", + "@webassemblyjs/helper-api-error": "1.13.2", "@xtuc/long": "4.2.2" } }, "node_modules/@webassemblyjs/helper-wasm-bytecode": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz", - "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==" + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.13.2.tgz", + "integrity": "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==", + "license": "MIT" }, "node_modules/@webassemblyjs/helper-wasm-section": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz", - "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.14.1.tgz", + "integrity": "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==", + "license": "MIT", "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-buffer": "1.12.1", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/wasm-gen": "1.12.1" + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/wasm-gen": "1.14.1" } }, "node_modules/@webassemblyjs/ieee754": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz", - "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.13.2.tgz", + "integrity": "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==", + "license": "MIT", "dependencies": { "@xtuc/ieee754": "^1.2.0" } }, "node_modules/@webassemblyjs/leb128": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz", - "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==", + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.13.2.tgz", + "integrity": "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==", + "license": "Apache-2.0", "dependencies": { "@xtuc/long": "4.2.2" } }, "node_modules/@webassemblyjs/utf8": { - "version": "1.11.6", - "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz", - "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==" + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.13.2.tgz", + "integrity": "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==", + "license": "MIT" }, "node_modules/@webassemblyjs/wasm-edit": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz", - "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.14.1.tgz", + "integrity": "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==", + "license": "MIT", "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-buffer": "1.12.1", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/helper-wasm-section": "1.12.1", - "@webassemblyjs/wasm-gen": "1.12.1", - "@webassemblyjs/wasm-opt": "1.12.1", - "@webassemblyjs/wasm-parser": "1.12.1", - "@webassemblyjs/wast-printer": "1.12.1" + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/helper-wasm-section": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-opt": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1", + "@webassemblyjs/wast-printer": "1.14.1" } }, "node_modules/@webassemblyjs/wasm-gen": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz", - "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.14.1.tgz", + "integrity": "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==", + "license": "MIT", "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/ieee754": "1.11.6", - "@webassemblyjs/leb128": "1.11.6", - "@webassemblyjs/utf8": "1.11.6" + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" } }, "node_modules/@webassemblyjs/wasm-opt": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz", - "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.14.1.tgz", + "integrity": "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==", + "license": "MIT", "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-buffer": "1.12.1", - "@webassemblyjs/wasm-gen": "1.12.1", - "@webassemblyjs/wasm-parser": "1.12.1" + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-buffer": "1.14.1", + "@webassemblyjs/wasm-gen": "1.14.1", + "@webassemblyjs/wasm-parser": "1.14.1" } }, "node_modules/@webassemblyjs/wasm-parser": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz", - "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz", + "integrity": "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==", + "license": "MIT", "dependencies": { - "@webassemblyjs/ast": "1.12.1", - "@webassemblyjs/helper-api-error": "1.11.6", - "@webassemblyjs/helper-wasm-bytecode": "1.11.6", - "@webassemblyjs/ieee754": "1.11.6", - "@webassemblyjs/leb128": "1.11.6", - "@webassemblyjs/utf8": "1.11.6" + "@webassemblyjs/ast": "1.14.1", + "@webassemblyjs/helper-api-error": "1.13.2", + "@webassemblyjs/helper-wasm-bytecode": "1.13.2", + "@webassemblyjs/ieee754": "1.13.2", + "@webassemblyjs/leb128": "1.13.2", + "@webassemblyjs/utf8": "1.13.2" } }, "node_modules/@webassemblyjs/wast-printer": { - "version": "1.12.1", - "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz", - "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==", + "version": "1.14.1", + "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.14.1.tgz", + "integrity": "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==", + "license": "MIT", "dependencies": { - "@webassemblyjs/ast": "1.12.1", + "@webassemblyjs/ast": "1.14.1", "@xtuc/long": "4.2.2" } }, @@ -5276,12 +5291,14 @@ "node_modules/@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", - "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==" + "integrity": "sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==", + "license": "BSD-3-Clause" }, "node_modules/@xtuc/long": { "version": "4.2.2", "resolved": "https://registry.npmjs.org/@xtuc/long/-/long-4.2.2.tgz", - "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==" + "integrity": "sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==", + "license": "Apache-2.0" }, "node_modules/abab": { "version": "2.0.6", @@ -6908,6 +6925,7 @@ "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", "dev": true, + "license": "MIT", "optional": true, "dependencies": { "file-uri-to-path": "1.0.0" @@ -11017,6 +11035,7 @@ "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", "dev": true, + "license": "MIT", "optional": true }, "node_modules/filename-regex": { @@ -18184,10 +18203,11 @@ "dev": true }, "node_modules/nan": { - "version": "2.19.0", - "resolved": "https://registry.npmjs.org/nan/-/nan-2.19.0.tgz", - "integrity": "sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==", + "version": "2.22.0", + "resolved": "https://registry.npmjs.org/nan/-/nan-2.22.0.tgz", + "integrity": "sha512-nbajikzWTMwsW+eSsNm3QwlOs7het9gGJU5dDZzRTQGk03vyBOauxgI4VakDzE0PtsGTmXPsXTbbjVhRwR5mpw==", "dev": true, + "license": "MIT", "optional": true }, "node_modules/nanoid": { @@ -22969,9 +22989,9 @@ } }, "node_modules/sass": { - "version": "1.81.1", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.81.1.tgz", - "integrity": "sha512-VNLgf4FC5yFyKwAumAAwwNh8X4SevlVREq3Y8aDZIkm0lI/zO1feycMXQ4hn+eB6FVhRbleSQ1Yb/q8juSldTA==", + "version": "1.82.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.82.0.tgz", + "integrity": "sha512-j4GMCTa8elGyN9A7x7bEglx0VgSpNUG4W4wNedQ33wSMdnkqQCT8HTwOaVSV4e6yQovcu/3Oc4coJP/l0xhL2Q==", "license": "MIT", "dependencies": { "chokidar": "^4.0.0", @@ -22989,9 +23009,9 @@ } }, "node_modules/sass-loader": { - "version": "16.0.3", - "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-16.0.3.tgz", - "integrity": "sha512-gosNorT1RCkuCMyihv6FBRR7BMV06oKRAs+l4UMp1mlcVg9rWN6KMmUj3igjQwmYys4mDP3etEYJgiHRbgHCHA==", + "version": "16.0.4", + "resolved": "https://registry.npmjs.org/sass-loader/-/sass-loader-16.0.4.tgz", + "integrity": "sha512-LavLbgbBGUt3wCiYzhuLLu65+fWXaXLmq7YxivLhEqmiupCFZ5sKUAipK3do6V80YSU0jvSxNhEdT13IXNr3rg==", "license": "MIT", "dependencies": { "neo-async": "^2.6.2" @@ -26292,16 +26312,16 @@ } }, "node_modules/webpack": { - "version": "5.96.1", - "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.96.1.tgz", - "integrity": "sha512-l2LlBSvVZGhL4ZrPwyr8+37AunkcYj5qh8o6u2/2rzoPc8gxFJkLj1WxNgooi9pnoc06jh0BjuXnamM4qlujZA==", + "version": "5.97.1", + "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.97.1.tgz", + "integrity": "sha512-EksG6gFY3L1eFMROS/7Wzgrii5mBAFe4rIr3r2BTfo7bcc+DWwFZ4OJ/miOuHJO/A85HwyI4eQ0F6IKXesO7Fg==", "license": "MIT", "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.6", - "@webassemblyjs/ast": "^1.12.1", - "@webassemblyjs/wasm-edit": "^1.12.1", - "@webassemblyjs/wasm-parser": "^1.12.1", + "@webassemblyjs/ast": "^1.14.1", + "@webassemblyjs/wasm-edit": "^1.14.1", + "@webassemblyjs/wasm-parser": "^1.14.1", "acorn": "^8.14.0", "browserslist": "^4.24.0", "chrome-trace-event": "^1.0.2", diff --git a/package.json b/package.json index 65d709e59021..2f09f8a7df90 100644 --- a/package.json +++ b/package.json @@ -12,14 +12,33 @@ "compile-sass-dev": "scripts/compile_sass.py --env=development", "watch": "{ npm run watch-webpack& npm run watch-sass& } && sleep infinity", "watch-webpack": "npm run webpack-dev -- --watch", - "watch-sass": "scripts/watch_sass.sh" + "watch-sass": "scripts/watch_sass.sh", + "lint": "python scripts/eslint.py", + "test": "npm run test-cms && npm run test-lms && npm run test-xmodule && npm run test-common && npm run test-jest", + "test-kind-vanilla": "npm run test-cms-vanilla && npm run test-xmodule-vanilla && npm run test-common-vanilla", + "test-kind-require": "npm run test-cms-require && npm run test-common-require", + "test-kind-webpack": "npm run test-cms-webpack && npm run test-lms-webpack && npm run test-xmodule-webpack", + "test-cms": "npm run test-cms-vanilla && npm run test-cms-require", + "test-cms-vanilla": "npm run test-suite -- cms/static/karma_cms.conf.js", + "test-cms-require": "npm run test-suite -- cms/static/karma_cms_squire.conf.js", + "test-cms-webpack": "npm run test-suite -- cms/static/karma_cms_webpack.conf.js", + "test-lms": "echo 'WARNING: Webpack JS tests are disabled. No LMS JS tests will be run. See https://github.com/openedx/edx-platform/issues/35956 for details.'", + "test-lms-webpack": "npm run test-suite -- lms/static/karma_lms.conf.js", + "test-xmodule": "npm run test-xmodule-vanilla", + "test-xmodule-vanilla": "npm run test-suite -- xmodule/js/karma_xmodule.conf.js", + "test-xmodule-webpack": "npm run test-suite -- xmodule/js/karma_xmodule_webpack.conf.js", + "test-common": "npm run test-common-vanilla && npm run test-common-require", + "test-common-vanilla": "npm run test-suite -- common/static/karma_common.conf.js", + "test-common-require": "npm run test-suite -- common/static/karma_common_requirejs.conf.js", + "test-suite": "${NODE_WRAPPER:-xvfb-run --auto-servernum} node --max_old_space_size=4096 node_modules/.bin/karma start --single-run=true --capture-timeout=60000 --browsers=FirefoxNoUpdates", + "test-jest": "jest" }, "dependencies": { "@babel/core": "7.26.0", "@babel/plugin-proposal-object-rest-spread": "^7.18.9", "@babel/plugin-transform-object-assign": "^7.18.6", "@babel/preset-env": "^7.19.0", - "@babel/preset-react": "7.25.9", + "@babel/preset-react": "7.26.3", "@edx/brand-edx.org": "^2.0.7", "@edx/edx-bootstrap": "1.0.4", "@edx/edx-proctoring": "^4.18.1", diff --git a/pavelib/__init__.py b/pavelib/__init__.py index 875068166ff5..24f05618bdd7 100644 --- a/pavelib/__init__.py +++ b/pavelib/__init__.py @@ -3,4 +3,4 @@ """ -from . import assets, js_test, prereqs, quality +from . import assets diff --git a/pavelib/js_test.py b/pavelib/js_test.py deleted file mode 100644 index fb9c213499ac..000000000000 --- a/pavelib/js_test.py +++ /dev/null @@ -1,143 +0,0 @@ -""" -Javascript test tasks -""" - - -import os -import re -import sys - -from paver.easy import cmdopts, needs, sh, task - -from pavelib.utils.envs import Env -from pavelib.utils.test.suites import JestSnapshotTestSuite, JsTestSuite -from pavelib.utils.timer import timed - -try: - from pygments.console import colorize -except ImportError: - colorize = lambda color, text: text - -__test__ = False # do not collect - - -@task -@needs( - 'pavelib.prereqs.install_node_prereqs', - 'pavelib.utils.test.utils.clean_reports_dir', -) -@cmdopts([ - ("suite=", "s", "Test suite to run"), - ("mode=", "m", "dev or run"), - ("coverage", "c", "Run test under coverage"), - ("port=", "p", "Port to run test server on (dev mode only)"), - ('skip-clean', 'C', 'skip cleaning repository before running tests'), - ('skip_clean', None, 'deprecated in favor of skip-clean'), -], share_with=["pavelib.utils.tests.utils.clean_reports_dir"]) -@timed -def test_js(options): - """ - Run the JavaScript tests - """ - mode = getattr(options, 'mode', 'run') - port = None - skip_clean = getattr(options, 'skip_clean', False) - - if mode == 'run': - suite = getattr(options, 'suite', 'all') - coverage = getattr(options, 'coverage', False) - elif mode == 'dev': - suite = getattr(options, 'suite', None) - coverage = False - port = getattr(options, 'port', None) - else: - sys.stderr.write("Invalid mode. Please choose 'dev' or 'run'.") - return - - if (suite != 'all') and (suite not in Env.JS_TEST_ID_KEYS): - sys.stderr.write( - "Unknown test suite. Please choose from ({suites})\n".format( - suites=", ".join(Env.JS_TEST_ID_KEYS) - ) - ) - return - - if suite != 'jest-snapshot': - test_suite = JsTestSuite(suite, mode=mode, with_coverage=coverage, port=port, skip_clean=skip_clean) - test_suite.run() - - if (suite == 'jest-snapshot') or (suite == 'all'): # lint-amnesty, pylint: disable=consider-using-in - test_suite = JestSnapshotTestSuite('jest') - test_suite.run() - - -@task -@cmdopts([ - ("suite=", "s", "Test suite to run"), - ("coverage", "c", "Run test under coverage"), -]) -@timed -def test_js_run(options): - """ - Run the JavaScript tests and print results to the console - """ - options.mode = 'run' - test_js(options) - - -@task -@cmdopts([ - ("suite=", "s", "Test suite to run"), - ("port=", "p", "Port to run test server on"), -]) -@timed -def test_js_dev(options): - """ - Run the JavaScript tests in your default browsers - """ - options.mode = 'dev' - test_js(options) - - -@task -@needs('pavelib.prereqs.install_coverage_prereqs') -@cmdopts([ - ("compare-branch=", "b", "Branch to compare against, defaults to origin/master"), -], share_with=['coverage']) -@timed -def diff_coverage(options): - """ - Build the diff coverage reports - """ - compare_branch = options.get('compare_branch', 'origin/master') - - # Find all coverage XML files (both Python and JavaScript) - xml_reports = [] - - for filepath in Env.REPORT_DIR.walk(): - if bool(re.match(r'^coverage.*\.xml$', filepath.basename())): - xml_reports.append(filepath) - - if not xml_reports: - err_msg = colorize( - 'red', - "No coverage info found. Run `paver test` before running " - "`paver coverage`.\n" - ) - sys.stderr.write(err_msg) - else: - xml_report_str = ' '.join(xml_reports) - diff_html_path = os.path.join(Env.REPORT_DIR, 'diff_coverage_combined.html') - - # Generate the diff coverage reports (HTML and console) - # The --diff-range-notation parameter is a workaround for https://github.com/Bachmann1234/diff_cover/issues/153 - sh( - "diff-cover {xml_report_str} --diff-range-notation '..' --compare-branch={compare_branch} " - "--html-report {diff_html_path}".format( - xml_report_str=xml_report_str, - compare_branch=compare_branch, - diff_html_path=diff_html_path, - ) - ) - - print("\n") diff --git a/pavelib/paver_tests/conftest.py b/pavelib/paver_tests/conftest.py deleted file mode 100644 index 214a35e3fe85..000000000000 --- a/pavelib/paver_tests/conftest.py +++ /dev/null @@ -1,22 +0,0 @@ -""" -Pytest fixtures for the pavelib unit tests. -""" - - -import os -from shutil import rmtree - -import pytest - -from pavelib.utils.envs import Env - - -@pytest.fixture(autouse=True, scope='session') -def delete_quality_junit_xml(): - """ - Delete the JUnit XML results files for quality check tasks run during the - unit tests. - """ - yield - if os.path.exists(Env.QUALITY_DIR): - rmtree(Env.QUALITY_DIR, ignore_errors=True) diff --git a/pavelib/paver_tests/test_eslint.py b/pavelib/paver_tests/test_eslint.py deleted file mode 100644 index 5802d7d0d21b..000000000000 --- a/pavelib/paver_tests/test_eslint.py +++ /dev/null @@ -1,54 +0,0 @@ -""" -Tests for Paver's Stylelint tasks. -""" - - -import unittest -from unittest.mock import patch - -import pytest -from paver.easy import BuildFailure, call_task - -import pavelib.quality - - -class TestPaverESLint(unittest.TestCase): - """ - For testing run_eslint - """ - - def setUp(self): - super().setUp() - - # Mock the paver @needs decorator - self._mock_paver_needs = patch.object(pavelib.quality.run_eslint, 'needs').start() - self._mock_paver_needs.return_value = 0 - - # Mock shell commands - patcher = patch('pavelib.quality.sh') - self._mock_paver_sh = patcher.start() - - # Cleanup mocks - self.addCleanup(patcher.stop) - self.addCleanup(self._mock_paver_needs.stop) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_count_from_last_line') - def test_eslint_violation_number_not_found(self, mock_count, mock_report_dir, mock_write_metric): # pylint: disable=unused-argument - """ - run_eslint encounters an error parsing the eslint output log - """ - mock_count.return_value = None - with pytest.raises(BuildFailure): - call_task('pavelib.quality.run_eslint', args=['']) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_count_from_last_line') - def test_eslint_vanilla(self, mock_count, mock_report_dir, mock_write_metric): # pylint: disable=unused-argument - """ - eslint finds violations, but a limit was not set - """ - mock_count.return_value = 1 - pavelib.quality.run_eslint("") diff --git a/pavelib/paver_tests/test_js_test.py b/pavelib/paver_tests/test_js_test.py deleted file mode 100644 index 4b165a156674..000000000000 --- a/pavelib/paver_tests/test_js_test.py +++ /dev/null @@ -1,148 +0,0 @@ -"""Unit tests for the Paver JavaScript testing tasks.""" - -from unittest.mock import patch - -import ddt -from paver.easy import call_task - -import pavelib.js_test -from pavelib.utils.envs import Env - -from .utils import PaverTestCase - - -@ddt.ddt -class TestPaverJavaScriptTestTasks(PaverTestCase): - """ - Test the Paver JavaScript testing tasks. - """ - - EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND = 'find {platform_root}/reports/javascript -type f -delete' - EXPECTED_KARMA_OPTIONS = ( - "{config_file} " - "--single-run={single_run} " - "--capture-timeout=60000 " - "--junitreportpath=" - "{platform_root}/reports/javascript/javascript_xunit-{suite}.xml " - "--browsers={browser}" - ) - EXPECTED_COVERAGE_OPTIONS = ( - ' --coverage --coveragereportpath={platform_root}/reports/javascript/coverage-{suite}.xml' - ) - - EXPECTED_COMMANDS = [ - "make report_dir", - 'git clean -fqdx test_root/logs test_root/data test_root/staticfiles test_root/uploads', - "find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \\;", - 'rm -rf test_root/log/auto_screenshots/*', - "rm -rf /tmp/mako_[cl]ms", - ] - - def setUp(self): - super().setUp() - - # Mock the paver @needs decorator - self._mock_paver_needs = patch.object(pavelib.js_test.test_js, 'needs').start() - self._mock_paver_needs.return_value = 0 - - # Cleanup mocks - self.addCleanup(self._mock_paver_needs.stop) - - @ddt.data( - [""], - ["--coverage"], - ["--suite=lms"], - ["--suite=lms --coverage"], - ) - @ddt.unpack - def test_test_js_run(self, options_string): - """ - Test the "test_js_run" task. - """ - options = self.parse_options_string(options_string) - self.reset_task_messages() - call_task("pavelib.js_test.test_js_run", options=options) - self.verify_messages(options=options, dev_mode=False) - - @ddt.data( - [""], - ["--port=9999"], - ["--suite=lms"], - ["--suite=lms --port=9999"], - ) - @ddt.unpack - def test_test_js_dev(self, options_string): - """ - Test the "test_js_run" task. - """ - options = self.parse_options_string(options_string) - self.reset_task_messages() - call_task("pavelib.js_test.test_js_dev", options=options) - self.verify_messages(options=options, dev_mode=True) - - def parse_options_string(self, options_string): - """ - Parse a string containing the options for a test run - """ - parameters = options_string.split(" ") - suite = "all" - if "--system=lms" in parameters: - suite = "lms" - elif "--system=common" in parameters: - suite = "common" - coverage = "--coverage" in parameters - port = None - if "--port=9999" in parameters: - port = 9999 - return { - "suite": suite, - "coverage": coverage, - "port": port, - } - - def verify_messages(self, options, dev_mode): - """ - Verify that the messages generated when running tests are as expected - for the specified options and dev_mode. - """ - is_coverage = options['coverage'] - port = options['port'] - expected_messages = [] - suites = Env.JS_TEST_ID_KEYS if options['suite'] == 'all' else [options['suite']] - - expected_messages.extend(self.EXPECTED_COMMANDS) - if not dev_mode and not is_coverage: - expected_messages.append(self.EXPECTED_DELETE_JAVASCRIPT_REPORT_COMMAND.format( - platform_root=self.platform_root - )) - - command_template = ( - 'node --max_old_space_size=4096 node_modules/.bin/karma start {options}' - ) - - for suite in suites: - # Karma test command - if suite != 'jest-snapshot': - karma_config_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(suite)] - expected_test_tool_command = command_template.format( - options=self.EXPECTED_KARMA_OPTIONS.format( - config_file=karma_config_file, - single_run='false' if dev_mode else 'true', - suite=suite, - platform_root=self.platform_root, - browser=Env.KARMA_BROWSER, - ), - ) - if is_coverage: - expected_test_tool_command += self.EXPECTED_COVERAGE_OPTIONS.format( - platform_root=self.platform_root, - suite=suite - ) - if port: - expected_test_tool_command += f" --port={port}" - else: - expected_test_tool_command = 'jest' - - expected_messages.append(expected_test_tool_command) - - assert self.task_messages == expected_messages diff --git a/pavelib/paver_tests/test_paver_quality.py b/pavelib/paver_tests/test_paver_quality.py deleted file mode 100644 index 36d6dd59e172..000000000000 --- a/pavelib/paver_tests/test_paver_quality.py +++ /dev/null @@ -1,156 +0,0 @@ -""" # lint-amnesty, pylint: disable=django-not-configured -Tests for paver quality tasks -""" - - -import os -import shutil # lint-amnesty, pylint: disable=unused-import -import tempfile -import textwrap -import unittest -from unittest.mock import MagicMock, mock_open, patch # lint-amnesty, pylint: disable=unused-import - -import pytest # lint-amnesty, pylint: disable=unused-import -from ddt import data, ddt, file_data, unpack # lint-amnesty, pylint: disable=unused-import -from path import Path as path -from paver.easy import BuildFailure # lint-amnesty, pylint: disable=unused-import - -import pavelib.quality -from pavelib.paver_tests.utils import PaverTestCase, fail_on_eslint # lint-amnesty, pylint: disable=unused-import - -OPEN_BUILTIN = 'builtins.open' - - -@ddt -class TestPaverQualityViolations(unittest.TestCase): - """ - For testing the paver violations-counting tasks - """ - def setUp(self): - super().setUp() - self.f = tempfile.NamedTemporaryFile(delete=False) # lint-amnesty, pylint: disable=consider-using-with - self.f.close() - self.addCleanup(os.remove, self.f.name) - - def test_pep8_parser(self): - with open(self.f.name, 'w') as f: - f.write("hello\nhithere") - num = len(pavelib.quality._pep8_violations(f.name)) # pylint: disable=protected-access - assert num == 2 - - -class TestPaverReportViolationsCounts(unittest.TestCase): - """ - For testing utility functions for getting counts from reports for - run_eslint and run_xsslint. - """ - - def setUp(self): - super().setUp() - - # Temporary file infrastructure - self.f = tempfile.NamedTemporaryFile(delete=False) # lint-amnesty, pylint: disable=consider-using-with - self.f.close() - - # Cleanup various mocks and tempfiles - self.addCleanup(os.remove, self.f.name) - - def test_get_eslint_violations_count(self): - with open(self.f.name, 'w') as f: - f.write("3000 violations found") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "eslint") # pylint: disable=protected-access - assert actual_count == 3000 - - def test_get_eslint_violations_no_number_found(self): - with open(self.f.name, 'w') as f: - f.write("Not expected string regex") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "eslint") # pylint: disable=protected-access - assert actual_count is None - - def test_get_eslint_violations_count_truncated_report(self): - """ - A truncated report (i.e. last line is just a violation) - """ - with open(self.f.name, 'w') as f: - f.write("foo/bar/js/fizzbuzz.js: line 45, col 59, Missing semicolon.") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "eslint") # pylint: disable=protected-access - assert actual_count is None - - def test_generic_value(self): - """ - Default behavior is to look for an integer appearing at head of line - """ - with open(self.f.name, 'w') as f: - f.write("5.777 good to see you") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "foo") # pylint: disable=protected-access - assert actual_count == 5 - - def test_generic_value_none_found(self): - """ - Default behavior is to look for an integer appearing at head of line - """ - with open(self.f.name, 'w') as f: - f.write("hello 5.777 good to see you") - actual_count = pavelib.quality._get_count_from_last_line(self.f.name, "foo") # pylint: disable=protected-access - assert actual_count is None - - def test_get_xsslint_counts_happy(self): - """ - Test happy path getting violation counts from xsslint report. - """ - report = textwrap.dedent(""" - test.html: 30:53: javascript-jquery-append: $('#test').append(print_tos); - - javascript-concat-html: 310 violations - javascript-escape: 7 violations - - 2608 violations total - """) - with open(self.f.name, 'w') as f: - f.write(report) - counts = pavelib.quality._get_xsslint_counts(self.f.name) # pylint: disable=protected-access - self.assertDictEqual(counts, { - 'rules': { - 'javascript-concat-html': 310, - 'javascript-escape': 7, - }, - 'total': 2608, - }) - - def test_get_xsslint_counts_bad_counts(self): - """ - Test getting violation counts from truncated and malformed xsslint - report. - """ - report = textwrap.dedent(""" - javascript-concat-html: violations - """) - with open(self.f.name, 'w') as f: - f.write(report) - counts = pavelib.quality._get_xsslint_counts(self.f.name) # pylint: disable=protected-access - self.assertDictEqual(counts, { - 'rules': {}, - 'total': None, - }) - - -class TestPrepareReportDir(unittest.TestCase): - """ - Tests the report directory preparation - """ - - def setUp(self): - super().setUp() - self.test_dir = tempfile.mkdtemp() - self.test_file = tempfile.NamedTemporaryFile(delete=False, dir=self.test_dir) # lint-amnesty, pylint: disable=consider-using-with - self.addCleanup(os.removedirs, self.test_dir) - - def test_report_dir_with_files(self): - assert os.path.exists(self.test_file.name) - pavelib.quality._prepare_report_dir(path(self.test_dir)) # pylint: disable=protected-access - assert not os.path.exists(self.test_file.name) - - def test_report_dir_without_files(self): - os.remove(self.test_file.name) - pavelib.quality._prepare_report_dir(path(self.test_dir)) # pylint: disable=protected-access - assert os.listdir(path(self.test_dir)) == [] diff --git a/pavelib/paver_tests/test_pii_check.py b/pavelib/paver_tests/test_pii_check.py deleted file mode 100644 index d034360acde0..000000000000 --- a/pavelib/paver_tests/test_pii_check.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -Tests for Paver's PII checker task. -""" - -import shutil -import tempfile -import unittest -from unittest.mock import patch - -from path import Path as path -from paver.easy import call_task, BuildFailure - -import pavelib.quality -from pavelib.utils.envs import Env - - -class TestPaverPIICheck(unittest.TestCase): - """ - For testing the paver run_pii_check task - """ - def setUp(self): - super().setUp() - self.report_dir = path(tempfile.mkdtemp()) - self.addCleanup(shutil.rmtree, self.report_dir) - - @patch.object(pavelib.quality.run_pii_check, 'needs') - @patch('pavelib.quality.sh') - def test_pii_check_report_dir_override(self, mock_paver_sh, mock_needs): - """ - run_pii_check succeeds with proper report dir - """ - # Make the expected stdout files. - cms_stdout_report = self.report_dir / 'pii_check_cms.report' - cms_stdout_report.write_lines(['Coverage found 33 uncovered models:\n']) - lms_stdout_report = self.report_dir / 'pii_check_lms.report' - lms_stdout_report.write_lines(['Coverage found 66 uncovered models:\n']) - - mock_needs.return_value = 0 - call_task('pavelib.quality.run_pii_check', options={"report_dir": str(self.report_dir)}) - mock_calls = [str(call) for call in mock_paver_sh.mock_calls] - assert len(mock_calls) == 2 - assert any('lms.envs.test' in call for call in mock_calls) - assert any('cms.envs.test' in call for call in mock_calls) - assert all(str(self.report_dir) in call for call in mock_calls) - metrics_file = Env.METRICS_DIR / 'pii' - assert open(metrics_file).read() == 'Number of PII Annotation violations: 66\n' - - @patch.object(pavelib.quality.run_pii_check, 'needs') - @patch('pavelib.quality.sh') - def test_pii_check_failed(self, mock_paver_sh, mock_needs): - """ - run_pii_check fails due to crossing the threshold. - """ - # Make the expected stdout files. - cms_stdout_report = self.report_dir / 'pii_check_cms.report' - cms_stdout_report.write_lines(['Coverage found 33 uncovered models:\n']) - lms_stdout_report = self.report_dir / 'pii_check_lms.report' - lms_stdout_report.write_lines([ - 'Coverage found 66 uncovered models:', - 'Coverage threshold not met! Needed 100.0, actually 95.0!', - ]) - - mock_needs.return_value = 0 - try: - with self.assertRaises(BuildFailure): - call_task('pavelib.quality.run_pii_check', options={"report_dir": str(self.report_dir)}) - except SystemExit: - # Sometimes the BuildFailure raises a SystemExit, sometimes it doesn't, not sure why. - # As a hack, we just wrap it in try-except. - # This is not good, but these tests weren't even running for years, and we're removing this whole test - # suite soon anyway. - pass - mock_calls = [str(call) for call in mock_paver_sh.mock_calls] - assert len(mock_calls) == 2 - assert any('lms.envs.test' in call for call in mock_calls) - assert any('cms.envs.test' in call for call in mock_calls) - assert all(str(self.report_dir) in call for call in mock_calls) - metrics_file = Env.METRICS_DIR / 'pii' - assert open(metrics_file).read() == 'Number of PII Annotation violations: 66\n' diff --git a/pavelib/paver_tests/test_stylelint.py b/pavelib/paver_tests/test_stylelint.py deleted file mode 100644 index 3e1c79c93f28..000000000000 --- a/pavelib/paver_tests/test_stylelint.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Tests for Paver's Stylelint tasks. -""" - -from unittest.mock import MagicMock, patch - -import pytest -import ddt -from paver.easy import call_task - -from .utils import PaverTestCase - - -@ddt.ddt -class TestPaverStylelint(PaverTestCase): - """ - Tests for Paver's Stylelint tasks. - """ - @ddt.data( - [False], - [True], - ) - @ddt.unpack - def test_run_stylelint(self, should_pass): - """ - Verify that the quality task fails with Stylelint violations. - """ - if should_pass: - _mock_stylelint_violations = MagicMock(return_value=0) - with patch('pavelib.quality._get_stylelint_violations', _mock_stylelint_violations): - call_task('pavelib.quality.run_stylelint') - else: - _mock_stylelint_violations = MagicMock(return_value=100) - with patch('pavelib.quality._get_stylelint_violations', _mock_stylelint_violations): - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_stylelint') diff --git a/pavelib/paver_tests/test_timer.py b/pavelib/paver_tests/test_timer.py deleted file mode 100644 index bc9817668347..000000000000 --- a/pavelib/paver_tests/test_timer.py +++ /dev/null @@ -1,168 +0,0 @@ -""" -Tests of the pavelib.utils.timer module. -""" - - -from datetime import datetime, timedelta -from unittest import TestCase - -from unittest.mock import MagicMock, patch - -from pavelib.utils import timer - - -@timer.timed -def identity(*args, **kwargs): - """ - An identity function used as a default task to test the timing of. - """ - return args, kwargs - - -MOCK_OPEN = MagicMock(spec=open) - - -@patch.dict('pavelib.utils.timer.__builtins__', open=MOCK_OPEN) -class TimedDecoratorTests(TestCase): - """ - Tests of the pavelib.utils.timer:timed decorator. - """ - def setUp(self): - super().setUp() - - patch_dumps = patch.object(timer.json, 'dump', autospec=True) - self.mock_dump = patch_dumps.start() - self.addCleanup(patch_dumps.stop) - - patch_makedirs = patch.object(timer.os, 'makedirs', autospec=True) - self.mock_makedirs = patch_makedirs.start() - self.addCleanup(patch_makedirs.stop) - - patch_datetime = patch.object(timer, 'datetime', autospec=True) - self.mock_datetime = patch_datetime.start() - self.addCleanup(patch_datetime.stop) - - patch_exists = patch.object(timer, 'exists', autospec=True) - self.mock_exists = patch_exists.start() - self.addCleanup(patch_exists.stop) - - MOCK_OPEN.reset_mock() - - def get_log_messages(self, task=identity, args=None, kwargs=None, raises=None): - """ - Return all timing messages recorded during the execution of ``task``. - """ - if args is None: - args = [] - if kwargs is None: - kwargs = {} - - if raises is None: - task(*args, **kwargs) - else: - self.assertRaises(raises, task, *args, **kwargs) - - return [ - call[0][0] # log_message - for call in self.mock_dump.call_args_list - ] - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_times(self): - start = datetime(2016, 7, 20, 10, 56, 19) - end = start + timedelta(seconds=35.6) - - self.mock_datetime.utcnow.side_effect = [start, end] - - messages = self.get_log_messages() - assert len(messages) == 1 - - assert 'duration' in messages[0] and messages[0]['duration'] == 35.6 - assert 'started_at' in messages[0] and messages[0]['started_at'] == start.isoformat(' ') - assert 'ended_at' in messages[0] and messages[0]['ended_at'] == end.isoformat(' ') - - @patch.object(timer, 'PAVER_TIMER_LOG', None) - def test_no_logs(self): - messages = self.get_log_messages() - assert len(messages) == 0 - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_arguments(self): - messages = self.get_log_messages(args=(1, 'foo'), kwargs=dict(bar='baz')) - assert len(messages) == 1 - - assert 'args' in messages[0] and messages[0]['args'] == [repr(1), repr('foo')] - assert 'kwargs' in messages[0] and messages[0]['kwargs'] == {'bar': repr('baz')} - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_task_name(self): - messages = self.get_log_messages() - assert len(messages) == 1 - - assert 'task' in messages[0] and messages[0]['task'] == 'pavelib.paver_tests.test_timer.identity' - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_exceptions(self): - @timer.timed - def raises(): - """ - A task used for testing exception handling of the timed decorator. - """ - raise Exception('The Message!') - - messages = self.get_log_messages(task=raises, raises=Exception) - assert len(messages) == 1 - - assert 'exception' in messages[0] and messages[0]['exception'] == 'Exception: The Message!' - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log-%Y-%m-%d-%H-%M-%S.log') - def test_date_formatting(self): - start = datetime(2016, 7, 20, 10, 56, 19) - end = start + timedelta(seconds=35.6) - - self.mock_datetime.utcnow.side_effect = [start, end] - - messages = self.get_log_messages() - assert len(messages) == 1 - - MOCK_OPEN.assert_called_once_with('/tmp/some-log-2016-07-20-10-56-19.log', 'a') - - @patch.object(timer, 'PAVER_TIMER_LOG', '/tmp/some-log') - def test_nested_tasks(self): - - @timer.timed - def parent(): - """ - A timed task that calls another task - """ - identity() - - parent_start = datetime(2016, 7, 20, 10, 56, 19) - parent_end = parent_start + timedelta(seconds=60) - child_start = parent_start + timedelta(seconds=10) - child_end = parent_end - timedelta(seconds=10) - - self.mock_datetime.utcnow.side_effect = [parent_start, child_start, child_end, parent_end] - - messages = self.get_log_messages(task=parent) - assert len(messages) == 2 - - # Child messages first - assert 'duration' in messages[0] - assert 40 == messages[0]['duration'] - - assert 'started_at' in messages[0] - assert child_start.isoformat(' ') == messages[0]['started_at'] - - assert 'ended_at' in messages[0] - assert child_end.isoformat(' ') == messages[0]['ended_at'] - - # Parent messages after - assert 'duration' in messages[1] - assert 60 == messages[1]['duration'] - - assert 'started_at' in messages[1] - assert parent_start.isoformat(' ') == messages[1]['started_at'] - - assert 'ended_at' in messages[1] - assert parent_end.isoformat(' ') == messages[1]['ended_at'] diff --git a/pavelib/paver_tests/test_xsslint.py b/pavelib/paver_tests/test_xsslint.py deleted file mode 100644 index a9b4a41e1600..000000000000 --- a/pavelib/paver_tests/test_xsslint.py +++ /dev/null @@ -1,120 +0,0 @@ -""" -Tests for paver xsslint quality tasks -""" -from unittest.mock import patch - -import pytest -from paver.easy import call_task - -import pavelib.quality - -from .utils import PaverTestCase - - -class PaverXSSLintTest(PaverTestCase): - """ - Test run_xsslint with a mocked environment in order to pass in opts - """ - - def setUp(self): - super().setUp() - self.reset_task_messages() - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_violation_number_not_found(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint encounters an error parsing the xsslint output log - """ - _mock_counts.return_value = {} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint') - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_vanilla(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds violations, but a limit was not set - """ - _mock_counts.return_value = {'total': 0} - call_task('pavelib.quality.run_xsslint') - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_invalid_thresholds_option(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint fails when thresholds option is poorly formatted - """ - _mock_counts.return_value = {'total': 0} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": "invalid"}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_invalid_thresholds_option_key(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint fails when thresholds option is poorly formatted - """ - _mock_counts.return_value = {'total': 0} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"invalid": 3}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_too_many_violations(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds more violations than are allowed - """ - _mock_counts.return_value = {'total': 4} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"total": 3}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_under_limit(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds fewer violations than are allowed - """ - _mock_counts.return_value = {'total': 4} - # No System Exit is expected - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"total": 5}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_rule_violation_number_not_found(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint encounters an error parsing the xsslint output log for a - given rule threshold that was set. - """ - _mock_counts.return_value = {'total': 4} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"rules": {"javascript-escape": 3}}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_too_many_rule_violations(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds more rule violations than are allowed - """ - _mock_counts.return_value = {'total': 4, 'rules': {'javascript-escape': 4}} - with pytest.raises(SystemExit): - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"rules": {"javascript-escape": 3}}'}) - - @patch.object(pavelib.quality, '_write_metric') - @patch.object(pavelib.quality, '_prepare_report_dir') - @patch.object(pavelib.quality, '_get_xsslint_counts') - def test_xsslint_under_rule_limit(self, _mock_counts, _mock_report_dir, _mock_write_metric): - """ - run_xsslint finds fewer rule violations than are allowed - """ - _mock_counts.return_value = {'total': 4, 'rules': {'javascript-escape': 4}} - # No System Exit is expected - call_task('pavelib.quality.run_xsslint', options={"thresholds": '{"rules": {"javascript-escape": 5}}'}) diff --git a/pavelib/quality.py b/pavelib/quality.py deleted file mode 100644 index 774179f45048..000000000000 --- a/pavelib/quality.py +++ /dev/null @@ -1,602 +0,0 @@ -""" # lint-amnesty, pylint: disable=django-not-configured -Check code quality using pycodestyle, pylint, and diff_quality. -""" - -import json -import os -import re -from datetime import datetime -from xml.sax.saxutils import quoteattr - -from paver.easy import BuildFailure, cmdopts, needs, sh, task - -from .utils.envs import Env -from .utils.timer import timed - -ALL_SYSTEMS = 'lms,cms,common,openedx,pavelib,scripts' -JUNIT_XML_TEMPLATE = """ - -{failure_element} - -""" -JUNIT_XML_FAILURE_TEMPLATE = '' -START_TIME = datetime.utcnow() - - -def write_junit_xml(name, message=None): - """ - Write a JUnit results XML file describing the outcome of a quality check. - """ - if message: - failure_element = JUNIT_XML_FAILURE_TEMPLATE.format(message=quoteattr(message)) - else: - failure_element = '' - data = { - 'failure_count': 1 if message else 0, - 'failure_element': failure_element, - 'name': name, - 'seconds': (datetime.utcnow() - START_TIME).total_seconds(), - } - Env.QUALITY_DIR.makedirs_p() - filename = Env.QUALITY_DIR / f'{name}.xml' - with open(filename, 'w') as f: - f.write(JUNIT_XML_TEMPLATE.format(**data)) - - -def fail_quality(name, message): - """ - Fail the specified quality check by generating the JUnit XML results file - and raising a ``BuildFailure``. - """ - write_junit_xml(name, message) - raise BuildFailure(message) - - -def top_python_dirs(dirname): - """ - Find the directories to start from in order to find all the Python files in `dirname`. - """ - top_dirs = [] - - dir_init = os.path.join(dirname, "__init__.py") - if os.path.exists(dir_init): - top_dirs.append(dirname) - - for directory in ['djangoapps', 'lib']: - subdir = os.path.join(dirname, directory) - subdir_init = os.path.join(subdir, "__init__.py") - if os.path.exists(subdir) and not os.path.exists(subdir_init): - dirs = os.listdir(subdir) - top_dirs.extend(d for d in dirs if os.path.isdir(os.path.join(subdir, d))) - - modules_to_remove = ['__pycache__'] - for module in modules_to_remove: - if module in top_dirs: - top_dirs.remove(module) - - return top_dirs - - -def _get_pep8_violations(clean=True): - """ - Runs pycodestyle. Returns a tuple of (number_of_violations, violations_string) - where violations_string is a string of all PEP 8 violations found, separated - by new lines. - """ - report_dir = (Env.REPORT_DIR / 'pep8') - if clean: - report_dir.rmtree(ignore_errors=True) - report_dir.makedirs_p() - report = report_dir / 'pep8.report' - - # Make sure the metrics subdirectory exists - Env.METRICS_DIR.makedirs_p() - - if not report.exists(): - sh(f'pycodestyle . | tee {report} -a') - - violations_list = _pep8_violations(report) - - return len(violations_list), violations_list - - -def _pep8_violations(report_file): - """ - Returns the list of all PEP 8 violations in the given report_file. - """ - with open(report_file) as f: - return f.readlines() - - -@task -@cmdopts([ - ("system=", "s", "System to act on"), -]) -@timed -def run_pep8(options): # pylint: disable=unused-argument - """ - Run pycodestyle on system code. - Fail the task if any violations are found. - """ - (count, violations_list) = _get_pep8_violations() - violations_list = ''.join(violations_list) - - # Print number of violations to log - violations_count_str = f"Number of PEP 8 violations: {count}" - print(violations_count_str) - print(violations_list) - - # Also write the number of violations to a file - with open(Env.METRICS_DIR / "pep8", "w") as f: - f.write(violations_count_str + '\n\n') - f.write(violations_list) - - # Fail if any violations are found - if count: - failure_string = "FAILURE: Too many PEP 8 violations. " + violations_count_str - failure_string += f"\n\nViolations:\n{violations_list}" - fail_quality('pep8', failure_string) - else: - write_junit_xml('pep8') - - -@task -@needs( - 'pavelib.prereqs.install_node_prereqs', - 'pavelib.utils.test.utils.ensure_clean_package_lock', -) -@cmdopts([ - ("limit=", "l", "limit for number of acceptable violations"), -]) -@timed -def run_eslint(options): - """ - Runs eslint on static asset directories. - If limit option is passed, fails build if more violations than the limit are found. - """ - - eslint_report_dir = (Env.REPORT_DIR / "eslint") - eslint_report = eslint_report_dir / "eslint.report" - _prepare_report_dir(eslint_report_dir) - violations_limit = int(getattr(options, 'limit', -1)) - - sh( - "node --max_old_space_size=4096 node_modules/.bin/eslint " - "--ext .js --ext .jsx --format=compact . | tee {eslint_report}".format( - eslint_report=eslint_report - ), - ignore_error=True - ) - - try: - num_violations = int(_get_count_from_last_line(eslint_report, "eslint")) - except TypeError: - fail_quality( - 'eslint', - "FAILURE: Number of eslint violations could not be found in {eslint_report}".format( - eslint_report=eslint_report - ) - ) - - # Record the metric - _write_metric(num_violations, (Env.METRICS_DIR / "eslint")) - - # Fail if number of violations is greater than the limit - if num_violations > violations_limit > -1: - fail_quality( - 'eslint', - "FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format( - count=num_violations, violations_limit=violations_limit - ) - ) - else: - write_junit_xml('eslint') - - -def _get_stylelint_violations(): - """ - Returns the number of Stylelint violations. - """ - stylelint_report_dir = (Env.REPORT_DIR / "stylelint") - stylelint_report = stylelint_report_dir / "stylelint.report" - _prepare_report_dir(stylelint_report_dir) - formatter = 'node_modules/stylelint-formatter-pretty' - - sh( - "stylelint **/*.scss --custom-formatter={formatter} | tee {stylelint_report}".format( - formatter=formatter, - stylelint_report=stylelint_report, - ), - ignore_error=True - ) - - try: - return int(_get_count_from_last_line(stylelint_report, "stylelint")) - except TypeError: - fail_quality( - 'stylelint', - "FAILURE: Number of stylelint violations could not be found in {stylelint_report}".format( - stylelint_report=stylelint_report - ) - ) - - -@task -@needs('pavelib.prereqs.install_node_prereqs') -@cmdopts([ - ("limit=", "l", "limit for number of acceptable violations"), -]) -@timed -def run_stylelint(options): - """ - Runs stylelint on Sass files. - If limit option is passed, fails build if more violations than the limit are found. - """ - violations_limit = 0 - num_violations = _get_stylelint_violations() - - # Record the metric - _write_metric(num_violations, (Env.METRICS_DIR / "stylelint")) - - # Fail if number of violations is greater than the limit - if num_violations > violations_limit: - fail_quality( - 'stylelint', - "FAILURE: Stylelint failed with too many violations: ({count}).\nThe limit is {violations_limit}.".format( - count=num_violations, - violations_limit=violations_limit, - ) - ) - else: - write_junit_xml('stylelint') - - -@task -@needs('pavelib.prereqs.install_python_prereqs') -@cmdopts([ - ("thresholds=", "t", "json containing limit for number of acceptable violations per rule"), -]) -@timed -def run_xsslint(options): - """ - Runs xsslint/xss_linter.py on the codebase - """ - - thresholds_option = getattr(options, 'thresholds', '{}') - try: - violation_thresholds = json.loads(thresholds_option) - except ValueError: - violation_thresholds = None - if isinstance(violation_thresholds, dict) is False or \ - any(key not in ("total", "rules") for key in violation_thresholds.keys()): - - fail_quality( - 'xsslint', - """FAILURE: Thresholds option "{thresholds_option}" was not supplied using proper format.\n""" - """Here is a properly formatted example, '{{"total":100,"rules":{{"javascript-escape":0}}}}' """ - """with property names in double-quotes.""".format( - thresholds_option=thresholds_option - ) - ) - - xsslint_script = "xss_linter.py" - xsslint_report_dir = (Env.REPORT_DIR / "xsslint") - xsslint_report = xsslint_report_dir / "xsslint.report" - _prepare_report_dir(xsslint_report_dir) - - sh( - "{repo_root}/scripts/xsslint/{xsslint_script} --rule-totals --config={cfg_module} >> {xsslint_report}".format( - repo_root=Env.REPO_ROOT, - xsslint_script=xsslint_script, - xsslint_report=xsslint_report, - cfg_module='scripts.xsslint_config' - ), - ignore_error=True - ) - - xsslint_counts = _get_xsslint_counts(xsslint_report) - - try: - metrics_str = "Number of {xsslint_script} violations: {num_violations}\n".format( - xsslint_script=xsslint_script, num_violations=int(xsslint_counts['total']) - ) - if 'rules' in xsslint_counts and any(xsslint_counts['rules']): - metrics_str += "\n" - rule_keys = sorted(xsslint_counts['rules'].keys()) - for rule in rule_keys: - metrics_str += "{rule} violations: {count}\n".format( - rule=rule, - count=int(xsslint_counts['rules'][rule]) - ) - except TypeError: - fail_quality( - 'xsslint', - "FAILURE: Number of {xsslint_script} violations could not be found in {xsslint_report}".format( - xsslint_script=xsslint_script, xsslint_report=xsslint_report - ) - ) - - metrics_report = (Env.METRICS_DIR / "xsslint") - # Record the metric - _write_metric(metrics_str, metrics_report) - # Print number of violations to log. - sh(f"cat {metrics_report}", ignore_error=True) - - error_message = "" - - # Test total violations against threshold. - if 'total' in list(violation_thresholds.keys()): - if violation_thresholds['total'] < xsslint_counts['total']: - error_message = "Too many violations total ({count}).\nThe limit is {violations_limit}.".format( - count=xsslint_counts['total'], violations_limit=violation_thresholds['total'] - ) - - # Test rule violations against thresholds. - if 'rules' in violation_thresholds: - threshold_keys = sorted(violation_thresholds['rules'].keys()) - for threshold_key in threshold_keys: - if threshold_key not in xsslint_counts['rules']: - error_message += ( - "\nNumber of {xsslint_script} violations for {rule} could not be found in " - "{xsslint_report}." - ).format( - xsslint_script=xsslint_script, rule=threshold_key, xsslint_report=xsslint_report - ) - elif violation_thresholds['rules'][threshold_key] < xsslint_counts['rules'][threshold_key]: - error_message += \ - "\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format( - rule=threshold_key, count=xsslint_counts['rules'][threshold_key], - violations_limit=violation_thresholds['rules'][threshold_key], - ) - - if error_message: - fail_quality( - 'xsslint', - "FAILURE: XSSLinter Failed.\n{error_message}\n" - "See {xsslint_report} or run the following command to hone in on the problem:\n" - " ./scripts/xss-commit-linter.sh -h".format( - error_message=error_message, xsslint_report=xsslint_report - ) - ) - else: - write_junit_xml('xsslint') - - -def _write_metric(metric, filename): - """ - Write a given metric to a given file - Used for things like reports/metrics/eslint, which will simply tell you the number of - eslint violations found - """ - Env.METRICS_DIR.makedirs_p() - - with open(filename, "w") as metric_file: - metric_file.write(str(metric)) - - -def _prepare_report_dir(dir_name): - """ - Sets a given directory to a created, but empty state - """ - dir_name.rmtree_p() - dir_name.mkdir_p() - - -def _get_report_contents(filename, report_name, last_line_only=False): - """ - Returns the contents of the given file. Use last_line_only to only return - the last line, which can be used for getting output from quality output - files. - - Arguments: - last_line_only: True to return the last line only, False to return a - string with full contents. - - Returns: - String containing full contents of the report, or the last line. - - """ - if os.path.isfile(filename): - with open(filename) as report_file: - if last_line_only: - lines = report_file.readlines() - for line in reversed(lines): - if line != '\n': - return line - return None - else: - return report_file.read() - else: - file_not_found_message = f"FAILURE: The following log file could not be found: {filename}" - fail_quality(report_name, file_not_found_message) - - -def _get_count_from_last_line(filename, file_type): - """ - This will return the number in the last line of a file. - It is returning only the value (as a floating number). - """ - report_contents = _get_report_contents(filename, file_type, last_line_only=True) - - if report_contents is None: - return 0 - - last_line = report_contents.strip() - # Example of the last line of a compact-formatted eslint report (for example): "62829 problems" - regex = r'^\d+' - - try: - return float(re.search(regex, last_line).group(0)) - # An AttributeError will occur if the regex finds no matches. - # A ValueError will occur if the returned regex cannot be cast as a float. - except (AttributeError, ValueError): - return None - - -def _get_xsslint_counts(filename): - """ - This returns a dict of violations from the xsslint report. - - Arguments: - filename: The name of the xsslint report. - - Returns: - A dict containing the following: - rules: A dict containing the count for each rule as follows: - violation-rule-id: N, where N is the number of violations - total: M, where M is the number of total violations - - """ - report_contents = _get_report_contents(filename, 'xsslint') - rule_count_regex = re.compile(r"^(?P[a-z-]+):\s+(?P\d+) violations", re.MULTILINE) - total_count_regex = re.compile(r"^(?P\d+) violations total", re.MULTILINE) - violations = {'rules': {}} - for violation_match in rule_count_regex.finditer(report_contents): - try: - violations['rules'][violation_match.group('rule_id')] = int(violation_match.group('count')) - except ValueError: - violations['rules'][violation_match.group('rule_id')] = None - try: - violations['total'] = int(total_count_regex.search(report_contents).group('count')) - # An AttributeError will occur if the regex finds no matches. - # A ValueError will occur if the returned regex cannot be cast as a float. - except (AttributeError, ValueError): - violations['total'] = None - return violations - - -def _extract_missing_pii_annotations(filename): - """ - Returns the number of uncovered models from the stdout report of django_find_annotations. - - Arguments: - filename: Filename where stdout of django_find_annotations was captured. - - Returns: - three-tuple containing: - 1. The number of uncovered models, - 2. A bool indicating whether the coverage is still below the threshold, and - 3. The full report as a string. - """ - uncovered_models = 0 - pii_check_passed = True - if os.path.isfile(filename): - with open(filename) as report_file: - lines = report_file.readlines() - - # Find the count of uncovered models. - uncovered_regex = re.compile(r'^Coverage found ([\d]+) uncovered') - for line in lines: - uncovered_match = uncovered_regex.match(line) - if uncovered_match: - uncovered_models = int(uncovered_match.groups()[0]) - break - - # Find a message which suggests the check failed. - failure_regex = re.compile(r'^Coverage threshold not met!') - for line in lines: - failure_match = failure_regex.match(line) - if failure_match: - pii_check_passed = False - break - - # Each line in lines already contains a newline. - full_log = ''.join(lines) - else: - fail_quality('pii', f'FAILURE: Log file could not be found: {filename}') - - return (uncovered_models, pii_check_passed, full_log) - - -@task -@needs('pavelib.prereqs.install_python_prereqs') -@cmdopts([ - ("report-dir=", "r", "Directory in which to put PII reports"), -]) -@timed -def run_pii_check(options): - """ - Guarantee that all Django models are PII-annotated. - """ - pii_report_name = 'pii' - default_report_dir = (Env.REPORT_DIR / pii_report_name) - report_dir = getattr(options, 'report_dir', default_report_dir) - output_file = os.path.join(report_dir, 'pii_check_{}.report') - env_report = [] - pii_check_passed = True - for env_name, env_settings_file in (("CMS", "cms.envs.test"), ("LMS", "lms.envs.test")): - try: - print() - print(f"Running {env_name} PII Annotation check and report") - print("-" * 45) - run_output_file = str(output_file).format(env_name.lower()) - sh( - "mkdir -p {} && " # lint-amnesty, pylint: disable=duplicate-string-formatting-argument - "export DJANGO_SETTINGS_MODULE={}; " - "code_annotations django_find_annotations " - "--config_file .pii_annotations.yml --report_path {} --app_name {} " - "--lint --report --coverage | tee {}".format( - report_dir, env_settings_file, report_dir, env_name.lower(), run_output_file - ) - ) - uncovered_model_count, pii_check_passed_env, full_log = _extract_missing_pii_annotations(run_output_file) - env_report.append(( - uncovered_model_count, - full_log, - )) - - except BuildFailure as error_message: - fail_quality(pii_report_name, f'FAILURE: {error_message}') - - if not pii_check_passed_env: - pii_check_passed = False - - # Determine which suite is the worst offender by obtaining the max() keying off uncovered_count. - uncovered_count, full_log = max(env_report, key=lambda r: r[0]) - - # Write metric file. - if uncovered_count is None: - uncovered_count = 0 - metrics_str = f"Number of PII Annotation violations: {uncovered_count}\n" - _write_metric(metrics_str, (Env.METRICS_DIR / pii_report_name)) - - # Finally, fail the paver task if code_annotations suggests that the check failed. - if not pii_check_passed: - fail_quality('pii', full_log) - - -@task -@needs('pavelib.prereqs.install_python_prereqs') -@timed -def check_keywords(): - """ - Check Django model fields for names that conflict with a list of reserved keywords - """ - report_path = os.path.join(Env.REPORT_DIR, 'reserved_keywords') - sh(f"mkdir -p {report_path}") - - overall_status = True - for env, env_settings_file in [('lms', 'lms.envs.test'), ('cms', 'cms.envs.test')]: - report_file = f"{env}_reserved_keyword_report.csv" - override_file = os.path.join(Env.REPO_ROOT, "db_keyword_overrides.yml") - try: - sh( - "export DJANGO_SETTINGS_MODULE={settings_file}; " - "python manage.py {app} check_reserved_keywords " - "--override_file {override_file} " - "--report_path {report_path} " - "--report_file {report_file}".format( - settings_file=env_settings_file, app=env, override_file=override_file, - report_path=report_path, report_file=report_file - ) - ) - except BuildFailure: - overall_status = False - - if not overall_status: - fail_quality( - 'keywords', - 'Failure: reserved keyword checker failed. Reports can be found here: {}'.format( - report_path - ) - ) diff --git a/pavelib/utils/test/suites/__init__.py b/pavelib/utils/test/suites/__init__.py deleted file mode 100644 index 34ecd49c1c74..000000000000 --- a/pavelib/utils/test/suites/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -""" -TestSuite class and subclasses -""" -from .js_suite import JestSnapshotTestSuite, JsTestSuite -from .suite import TestSuite diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py deleted file mode 100644 index 4e53d454fee5..000000000000 --- a/pavelib/utils/test/suites/js_suite.py +++ /dev/null @@ -1,109 +0,0 @@ -""" -Javascript test tasks -""" - - -from paver import tasks - -from pavelib.utils.envs import Env -from pavelib.utils.test import utils as test_utils -from pavelib.utils.test.suites.suite import TestSuite - -__test__ = False # do not collect - - -class JsTestSuite(TestSuite): - """ - A class for running JavaScript tests. - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.run_under_coverage = kwargs.get('with_coverage', True) - self.mode = kwargs.get('mode', 'run') - self.report_dir = Env.JS_REPORT_DIR - self.opts = kwargs - - suite = args[0] - self.subsuites = self._default_subsuites if suite == 'all' else [JsTestSubSuite(*args, **kwargs)] - - def __enter__(self): - super().__enter__() - if tasks.environment.dry_run: - tasks.environment.info("make report_dir") - else: - self.report_dir.makedirs_p() - if not self.skip_clean: - test_utils.clean_test_files() - - if self.mode == 'run' and not self.run_under_coverage: - test_utils.clean_dir(self.report_dir) - - @property - def _default_subsuites(self): - """ - Returns all JS test suites - """ - return [JsTestSubSuite(test_id, **self.opts) for test_id in Env.JS_TEST_ID_KEYS if test_id != 'jest-snapshot'] - - -class JsTestSubSuite(TestSuite): - """ - Class for JS suites like cms, cms-squire, lms, common, - common-requirejs and xmodule - """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.test_id = args[0] - self.run_under_coverage = kwargs.get('with_coverage', True) - self.mode = kwargs.get('mode', 'run') - self.port = kwargs.get('port') - self.root = self.root + ' javascript' - self.report_dir = Env.JS_REPORT_DIR - - try: - self.test_conf_file = Env.KARMA_CONFIG_FILES[Env.JS_TEST_ID_KEYS.index(self.test_id)] - except ValueError: - self.test_conf_file = Env.KARMA_CONFIG_FILES[0] - - self.coverage_report = self.report_dir / f'coverage-{self.test_id}.xml' - self.xunit_report = self.report_dir / f'javascript_xunit-{self.test_id}.xml' - - @property - def cmd(self): - """ - Run the tests using karma runner. - """ - cmd = [ - "node", - "--max_old_space_size=4096", - "node_modules/.bin/karma", - "start", - self.test_conf_file, - "--single-run={}".format('false' if self.mode == 'dev' else 'true'), - "--capture-timeout=60000", - f"--junitreportpath={self.xunit_report}", - f"--browsers={Env.KARMA_BROWSER}", - ] - - if self.port: - cmd.append(f"--port={self.port}") - - if self.run_under_coverage: - cmd.extend([ - "--coverage", - f"--coveragereportpath={self.coverage_report}", - ]) - - return cmd - - -class JestSnapshotTestSuite(TestSuite): - """ - A class for running Jest Snapshot tests. - """ - @property - def cmd(self): - """ - Run the tests using Jest. - """ - return ["jest"] diff --git a/pavelib/utils/test/suites/suite.py b/pavelib/utils/test/suites/suite.py deleted file mode 100644 index 5a423c827c21..000000000000 --- a/pavelib/utils/test/suites/suite.py +++ /dev/null @@ -1,149 +0,0 @@ -""" -A class used for defining and running test suites -""" - - -import os -import subprocess -import sys - -from paver import tasks - -from pavelib.utils.process import kill_process - -try: - from pygments.console import colorize -except ImportError: - colorize = lambda color, text: text - -__test__ = False # do not collect - - -class TestSuite: - """ - TestSuite is a class that defines how groups of tests run. - """ - def __init__(self, *args, **kwargs): - self.root = args[0] - self.subsuites = kwargs.get('subsuites', []) - self.failed_suites = [] - self.verbosity = int(kwargs.get('verbosity', 1)) - self.skip_clean = kwargs.get('skip_clean', False) - self.passthrough_options = kwargs.get('passthrough_options', []) - - def __enter__(self): - """ - This will run before the test suite is run with the run_suite_tests method. - If self.run_test is called directly, it should be run in a 'with' block to - ensure that the proper context is created. - - Specific setup tasks should be defined in each subsuite. - - i.e. Checking for and defining required directories. - """ - print(f"\nSetting up for {self.root}") - self.failed_suites = [] - - def __exit__(self, exc_type, exc_value, traceback): - """ - This is run after the tests run with the run_suite_tests method finish. - Specific clean up tasks should be defined in each subsuite. - - If self.run_test is called directly, it should be run in a 'with' block - to ensure that clean up happens properly. - - i.e. Cleaning mongo after the lms tests run. - """ - print(f"\nCleaning up after {self.root}") - - @property - def cmd(self): - """ - The command to run tests (as a string). For this base class there is none. - """ - return None - - @staticmethod - def is_success(exit_code): - """ - Determine if the given exit code represents a success of the test - suite. By default, only a zero counts as a success. - """ - return exit_code == 0 - - def run_test(self): - """ - Runs a self.cmd in a subprocess and waits for it to finish. - It returns False if errors or failures occur. Otherwise, it - returns True. - """ - cmd = " ".join(self.cmd) - - if tasks.environment.dry_run: - tasks.environment.info(cmd) - return - - sys.stdout.write(cmd) - - msg = colorize( - 'green', - '\n{bar}\n Running tests for {suite_name} \n{bar}\n'.format(suite_name=self.root, bar='=' * 40), - ) - - sys.stdout.write(msg) - sys.stdout.flush() - - if 'TEST_SUITE' not in os.environ: - os.environ['TEST_SUITE'] = self.root.replace("/", "_") - kwargs = {'shell': True, 'cwd': None} - process = None - - try: - process = subprocess.Popen(cmd, **kwargs) # lint-amnesty, pylint: disable=consider-using-with - return self.is_success(process.wait()) - except KeyboardInterrupt: - kill_process(process) - sys.exit(1) - - def run_suite_tests(self): - """ - Runs each of the suites in self.subsuites while tracking failures - """ - # Uses __enter__ and __exit__ for context - with self: - # run the tests for this class, and for all subsuites - if self.cmd: - passed = self.run_test() - if not passed: - self.failed_suites.append(self) - - for suite in self.subsuites: - suite.run_suite_tests() - if suite.failed_suites: - self.failed_suites.extend(suite.failed_suites) - - def report_test_results(self): - """ - Writes a list of failed_suites to sys.stderr - """ - if self.failed_suites: - msg = colorize('red', "\n\n{bar}\nTests failed in the following suites:\n* ".format(bar="=" * 48)) - msg += colorize('red', '\n* '.join([s.root for s in self.failed_suites]) + '\n\n') - else: - msg = colorize('green', "\n\n{bar}\nNo test failures ".format(bar="=" * 48)) - - print(msg) - - def run(self): - """ - Runs the tests in the suite while tracking and reporting failures. - """ - self.run_suite_tests() - - if tasks.environment.dry_run: - return - - self.report_test_results() - - if self.failed_suites: - sys.exit(1) diff --git a/pavelib/utils/test/utils.py b/pavelib/utils/test/utils.py deleted file mode 100644 index 0851251e2222..000000000000 --- a/pavelib/utils/test/utils.py +++ /dev/null @@ -1,91 +0,0 @@ -""" -Helper functions for test tasks -""" - - -import os - -from paver.easy import cmdopts, sh, task - -from pavelib.utils.envs import Env -from pavelib.utils.timer import timed - - -MONGO_PORT_NUM = int(os.environ.get('EDXAPP_TEST_MONGO_PORT', '27017')) - -COVERAGE_CACHE_BUCKET = "edx-tools-coverage-caches" -COVERAGE_CACHE_BASEPATH = "test_root/who_tests_what" -COVERAGE_CACHE_BASELINE = "who_tests_what.{}.baseline".format(os.environ.get('WTW_CONTEXT', 'all')) -WHO_TESTS_WHAT_DIFF = "who_tests_what.diff" - - -__test__ = False # do not collect - - -@task -@timed -def clean_test_files(): - """ - Clean fixture files used by tests and .pyc files - """ - sh("git clean -fqdx test_root/logs test_root/data test_root/staticfiles test_root/uploads") - # This find command removes all the *.pyc files that aren't in the .git - # directory. See this blog post for more details: - # http://nedbatchelder.com/blog/201505/be_careful_deleting_files_around_git.html - sh(r"find . -name '.git' -prune -o -name '*.pyc' -exec rm {} \;") - sh("rm -rf test_root/log/auto_screenshots/*") - sh("rm -rf /tmp/mako_[cl]ms") - - -@task -@timed -def ensure_clean_package_lock(): - """ - Ensure no untracked changes have been made in the current git context. - """ - sh(""" - git diff --name-only --exit-code package-lock.json || - (echo \"Dirty package-lock.json, run 'npm install' and commit the generated changes\" && exit 1) - """) - - -def clean_dir(directory): - """ - Delete all the files from the specified directory. - """ - # We delete the files but preserve the directory structure - # so that coverage.py has a place to put the reports. - sh(f'find {directory} -type f -delete') - - -@task -@cmdopts([ - ('skip-clean', 'C', 'skip cleaning repository before running tests'), - ('skip_clean', None, 'deprecated in favor of skip-clean'), -]) -@timed -def clean_reports_dir(options): - """ - Clean coverage files, to ensure that we don't use stale data to generate reports. - """ - if getattr(options, 'skip_clean', False): - print('--skip-clean is set, skipping...') - return - - # We delete the files but preserve the directory structure - # so that coverage.py has a place to put the reports. - reports_dir = Env.REPORT_DIR.makedirs_p() - clean_dir(reports_dir) - - -@task -@timed -def clean_mongo(): - """ - Clean mongo test databases - """ - sh("mongo {host}:{port} {repo_root}/scripts/delete-mongo-test-dbs.js".format( - host=Env.MONGO_HOST, - port=MONGO_PORT_NUM, - repo_root=Env.REPO_ROOT, - )) diff --git a/requirements/constraints.txt b/requirements/constraints.txt index 991291fa1ab1..d7608873644e 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -78,7 +78,7 @@ django-storages<1.14.4 # The team that owns this package will manually bump this package rather than having it pulled in automatically. # This is to allow them to better control its deployment and to do it in a process that works better # for them. -edx-enterprise==5.3.1 +edx-enterprise==5.4.2 # Date: 2024-05-09 # This has to be constrained as well because newer versions of edx-i18n-tools need the diff --git a/requirements/edx-sandbox/base.txt b/requirements/edx-sandbox/base.txt index 8cf0cbb82904..60a5eff8a62d 100644 --- a/requirements/edx-sandbox/base.txt +++ b/requirements/edx-sandbox/base.txt @@ -14,13 +14,13 @@ click==8.1.6 # nltk codejail-includes==1.0.0 # via -r requirements/edx-sandbox/base.in -contourpy==1.3.0 +contourpy==1.3.1 # via matplotlib -cryptography==43.0.3 +cryptography==44.0.0 # via -r requirements/edx-sandbox/base.in cycler==0.12.1 # via matplotlib -fonttools==4.54.1 +fonttools==4.55.2 # via matplotlib joblib==1.4.2 # via nltk @@ -31,13 +31,13 @@ lxml[html-clean,html_clean]==5.3.0 # -r requirements/edx-sandbox/base.in # lxml-html-clean # openedx-calc -lxml-html-clean==0.3.1 +lxml-html-clean==0.4.1 # via lxml markupsafe==3.0.2 # via # chem # openedx-calc -matplotlib==3.9.2 +matplotlib==3.9.3 # via -r requirements/edx-sandbox/base.in mpmath==1.3.0 # via sympy @@ -55,9 +55,9 @@ numpy==1.26.4 # matplotlib # openedx-calc # scipy -openedx-calc==3.1.2 +openedx-calc==4.0.1 # via -r requirements/edx-sandbox/base.in -packaging==24.1 +packaging==24.2 # via matplotlib pillow==11.0.0 # via matplotlib @@ -73,14 +73,14 @@ python-dateutil==2.9.0.post0 # via matplotlib random2==1.0.2 # via -r requirements/edx-sandbox/base.in -regex==2024.9.11 +regex==2024.11.6 # via nltk scipy==1.14.1 # via # -r requirements/edx-sandbox/base.in # chem # openedx-calc -six==1.16.0 +six==1.17.0 # via # codejail-includes # python-dateutil @@ -88,5 +88,5 @@ sympy==1.13.3 # via # -r requirements/edx-sandbox/base.in # openedx-calc -tqdm==4.66.6 +tqdm==4.67.1 # via nltk diff --git a/requirements/edx/assets.txt b/requirements/edx/assets.txt index 6c3e1a41515c..15c38017919c 100644 --- a/requirements/edx/assets.txt +++ b/requirements/edx/assets.txt @@ -14,5 +14,5 @@ libsass==0.10.0 # -r requirements/edx/assets.in nodeenv==1.9.1 # via -r requirements/edx/assets.in -six==1.16.0 +six==1.17.0 # via libsass diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 568c9cfd6361..c6e3d5fd1e67 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -8,9 +8,9 @@ # via -r requirements/edx/github.in acid-xblock==0.4.1 # via -r requirements/edx/kernel.in -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.11.9 # via # geoip2 # openai @@ -20,7 +20,7 @@ algoliasearch==3.0.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/bundled.in -amqp==5.2.0 +amqp==5.3.1 # via kombu analytics-python==1.4.post1 # via -r requirements/edx/kernel.in @@ -54,7 +54,7 @@ babel==2.16.0 # enmerkar-underscore backoff==1.10.0 # via analytics-python -bcrypt==4.2.0 +bcrypt==4.2.1 # via paramiko beautifulsoup4==4.12.3 # via @@ -62,7 +62,7 @@ beautifulsoup4==4.12.3 # pynliner billiard==4.2.1 # via celery -bleach[css]==6.1.0 +bleach[css]==6.2.0 # via # edx-enterprise # lti-consumer-xblock @@ -72,20 +72,20 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/kernel.in -boto3==1.35.50 +boto3==1.35.76 # via # -r requirements/edx/kernel.in # django-ses # fs-s3fs # ora2 -botocore==1.35.50 +botocore==1.35.76 # via # -r requirements/edx/kernel.in # boto3 # s3transfer bridgekeeper==0.9 # via -r requirements/edx/kernel.in -cachecontrol==0.14.0 +cachecontrol==0.14.1 # via firebase-admin cachetools==5.5.0 # via google-auth @@ -140,7 +140,7 @@ click-plugins==1.1.1 # via celery click-repl==0.3.0 # via celery -code-annotations==1.8.0 +code-annotations==2.0.0 # via # edx-enterprise # edx-toggles @@ -148,7 +148,7 @@ codejail-includes==1.0.0 # via -r requirements/edx/kernel.in crowdsourcehinter-xblock==0.8 # via -r requirements/edx/bundled.in -cryptography==43.0.3 +cryptography==44.0.0 # via # -r requirements/edx/kernel.in # django-fernet-fields-v2 @@ -170,7 +170,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==4.2.16 +django==4.2.17 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -257,7 +257,7 @@ django-config-models==2.7.0 # edx-enterprise # edx-name-affirmation # lti-consumer-xblock -django-cors-headers==4.5.0 +django-cors-headers==4.6.0 # via -r requirements/edx/kernel.in django-countries==7.6.1 # via @@ -312,7 +312,7 @@ django-mptt==0.16.0 # openedx-django-wiki django-multi-email-field==0.7.0 # via edx-enterprise -django-mysql==4.14.0 +django-mysql==4.15.0 # via -r requirements/edx/kernel.in django-oauth-toolkit==1.7.1 # via @@ -331,7 +331,7 @@ django-sekizai==4.1.0 # via # -r requirements/edx/kernel.in # openedx-django-wiki -django-ses==4.2.0 +django-ses==4.3.0 # via -r requirements/edx/bundled.in django-simple-history==3.4.0 # via @@ -342,7 +342,7 @@ django-simple-history==3.4.0 # edx-organizations # edx-proctoring # ora2 -django-statici18n==2.5.0 +django-statici18n==2.6.0 # via # -r requirements/edx/kernel.in # lti-consumer-xblock @@ -356,7 +356,7 @@ django-storages==1.14.3 # edxval django-user-tasks==3.2.0 # via -r requirements/edx/kernel.in -django-waffle==4.1.0 +django-waffle==4.2.0 # via # -r requirements/edx/kernel.in # edx-django-utils @@ -400,7 +400,7 @@ done-xblock==2.4.0 # via -r requirements/edx/bundled.in drf-jwt==1.19.2 # via edx-drf-extensions -drf-spectacular==0.27.2 +drf-spectacular==0.28.0 # via -r requirements/edx/kernel.in drf-yasg==1.21.8 # via @@ -422,7 +422,7 @@ edx-bulk-grades==1.1.0 # via # -r requirements/edx/kernel.in # staff-graded-xblock -edx-ccx-keys==1.3.0 +edx-ccx-keys==2.0.2 # via # -r requirements/edx/kernel.in # lti-consumer-xblock @@ -434,7 +434,7 @@ edx-celeryutils==1.3.0 # super-csv edx-codejail==3.5.2 # via -r requirements/edx/kernel.in -edx-completion==4.7.3 +edx-completion==4.7.6 # via -r requirements/edx/kernel.in edx-django-release-util==1.4.0 # via @@ -443,7 +443,7 @@ edx-django-release-util==1.4.0 # edxval edx-django-sites-extensions==4.2.0 # via -r requirements/edx/kernel.in -edx-django-utils==7.0.0 +edx-django-utils==7.1.0 # via # -r requirements/edx/kernel.in # django-config-models @@ -472,7 +472,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.3.1 +edx-enterprise==5.4.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/kernel.in @@ -508,7 +508,7 @@ edx-opaque-keys[django]==2.11.0 # ora2 edx-organizations==6.13.0 # via -r requirements/edx/kernel.in -edx-proctoring==4.18.3 +edx-proctoring==4.18.4 # via # -r requirements/edx/kernel.in # edx-proctoring-proctortrack @@ -525,7 +525,7 @@ edx-search==4.1.1 # openedx-forum edx-sga==0.25.0 # via -r requirements/edx/bundled.in -edx-submissions==3.8.2 +edx-submissions==3.8.3 # via # -r requirements/edx/kernel.in # ora2 @@ -549,7 +549,7 @@ edx-when==2.5.0 # via # -r requirements/edx/kernel.in # edx-proctoring -edxval==2.6.0 +edxval==2.6.1 # via -r requirements/edx/kernel.in elasticsearch==7.9.1 # via @@ -571,7 +571,7 @@ fastavro==1.9.7 # via openedx-events filelock==3.16.1 # via snowflake-connector-python -firebase-admin==6.5.0 +firebase-admin==6.6.0 # via edx-ace frozenlist==1.5.0 # via @@ -589,20 +589,20 @@ fs-s3fs==0.1.8 # openedx-django-pyfs future==1.0.0 # via pyjwkest -geoip2==4.8.0 +geoip2==4.8.1 # via -r requirements/edx/kernel.in glob2==0.7 # via -r requirements/edx/kernel.in -google-api-core[grpc]==2.22.0 +google-api-core[grpc]==2.23.0 # via # firebase-admin # google-api-python-client # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.149.0 +google-api-python-client==2.154.0 # via firebase-admin -google-auth==2.35.0 +google-auth==2.36.0 # via # google-api-core # google-api-python-client @@ -618,7 +618,7 @@ google-cloud-core==2.4.1 # google-cloud-storage google-cloud-firestore==2.19.0 # via firebase-admin -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via firebase-admin google-crc32c==1.6.0 # via @@ -626,15 +626,15 @@ google-crc32c==1.6.0 # google-resumable-media google-resumable-media==2.7.2 # via google-cloud-storage -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # google-api-core # grpcio-status -grpcio==1.67.0 +grpcio==1.68.1 # via # google-api-core # grpcio-status -grpcio-status==1.67.0 +grpcio-status==1.68.1 # via google-api-core gunicorn==23.0.0 # via -r requirements/edx/kernel.in @@ -648,7 +648,7 @@ httplib2==0.22.0 # via # google-api-python-client # google-auth-httplib2 -icalendar==6.0.1 +icalendar==6.1.0 # via -r requirements/edx/kernel.in idna==3.10 # via @@ -715,7 +715,7 @@ libsass==0.10.0 # -r requirements/edx/paver.txt loremipsum==1.0.5 # via ora2 -lti-consumer-xblock==9.11.3 +lti-consumer-xblock==9.12.0 # via -r requirements/edx/kernel.in lxml[html-clean,html_clean]==5.3.0 # via @@ -730,11 +730,11 @@ lxml[html-clean,html_clean]==5.3.0 # python3-saml # xblock # xmlsec -lxml-html-clean==0.3.1 +lxml-html-clean==0.4.1 # via lxml mailsnake==1.6.4 # via -r requirements/edx/bundled.in -mako==1.3.6 +mako==1.3.7 # via # -r requirements/edx/kernel.in # acid-xblock @@ -758,7 +758,7 @@ markupsafe==3.0.2 # xblock maxminddb==2.6.2 # via geoip2 -meilisearch==0.31.6 +meilisearch==0.33.0 # via # -r requirements/edx/kernel.in # edx-search @@ -780,13 +780,13 @@ multidict==6.1.0 # via # aiohttp # yarl -mysqlclient==2.2.5 +mysqlclient==2.2.6 # via # -r requirements/edx/kernel.in # openedx-forum -newrelic==10.2.0 +newrelic==10.3.1 # via edx-django-utils -nh3==0.2.18 +nh3==0.2.19 # via -r requirements/edx/kernel.in nltk==3.9.1 # via chem @@ -816,7 +816,7 @@ openedx-atlas==0.6.2 # via # -r requirements/edx/kernel.in # openedx-forum -openedx-calc==3.1.2 +openedx-calc==4.0.1 # via -r requirements/edx/kernel.in openedx-django-pyfs==3.7.0 # via @@ -841,7 +841,7 @@ openedx-filters==1.11.0 # -r requirements/edx/kernel.in # lti-consumer-xblock # ora2 -openedx-forum==0.1.3 +openedx-forum==0.1.5 # via -r requirements/edx/kernel.in openedx-learning==0.18.1 # via @@ -855,13 +855,13 @@ optimizely-sdk==4.1.1 # -r requirements/edx/bundled.in ora2==6.14.1 # via -r requirements/edx/bundled.in -packaging==24.1 +packaging==24.2 # via # drf-yasg # gunicorn # py2neo # snowflake-connector-python -pansi==2020.7.3 +pansi==2024.11.0 # via py2neo paramiko==3.5.0 # via edx-enterprise @@ -893,19 +893,22 @@ pillow==11.0.0 # edx-enterprise # edx-organizations # edxval + # pansi platformdirs==4.3.6 # via snowflake-connector-python polib==1.2.0 # via edx-i18n-tools prompt-toolkit==3.0.48 # via click-repl -propcache==0.2.0 - # via yarl +propcache==0.2.1 + # via + # aiohttp + # yarl proto-plus==1.25.0 # via # google-api-core # google-cloud-firestore -protobuf==5.28.3 +protobuf==5.29.1 # via # google-api-core # google-cloud-firestore @@ -937,9 +940,9 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.9.2 +pydantic==2.10.3 # via camel-converter -pydantic-core==2.23.4 +pydantic-core==2.27.1 # via pydantic pygments==2.18.0 # via @@ -950,7 +953,7 @@ pyjwkest==1.4.2 # -r requirements/edx/kernel.in # edx-token-utils # lti-consumer-xblock -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.1 # via # -r requirements/edx/kernel.in # drf-jwt @@ -984,7 +987,7 @@ pynacl==1.5.0 # paramiko pynliner==0.8.0 # via -r requirements/edx/kernel.in -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # optimizely-sdk # snowflake-connector-python @@ -1067,7 +1070,7 @@ referencing==0.35.1 # via # jsonschema # jsonschema-specifications -regex==2024.9.11 +regex==2024.11.6 # via nltk requests==2.32.3 # via @@ -1101,7 +1104,7 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/kernel.in # social-auth-core -rpds-py==0.20.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -1113,7 +1116,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.10.3 +s3transfer==0.10.4 # via boto3 sailthru-client==2.2.3 # via edx-ace @@ -1132,12 +1135,11 @@ simplejson==3.19.3 # super-csv # xblock # xblock-utils -six==1.16.0 +six==1.17.0 # via # -r requirements/edx/kernel.in # -r requirements/edx/paver.txt # analytics-python - # bleach # codejail-includes # crowdsourcehinter-xblock # edx-ace @@ -1154,7 +1156,6 @@ six==1.16.0 # interchange # libsass # optimizely-sdk - # pansi # paver # py2neo # pyjwkest @@ -1164,7 +1165,7 @@ slumber==0.7.1 # -r requirements/edx/kernel.in # edx-bulk-grades # edx-enterprise -snowflake-connector-python==3.12.3 +snowflake-connector-python==3.12.4 # via edx-enterprise social-auth-app-django==5.4.1 # via @@ -1186,11 +1187,11 @@ sortedcontainers==2.4.0 # snowflake-connector-python soupsieve==2.6 # via beautifulsoup4 -sqlparse==0.5.1 +sqlparse==0.5.2 # via django staff-graded-xblock==2.3.0 # via -r requirements/edx/bundled.in -stevedore==5.3.0 +stevedore==5.4.0 # via # -r requirements/edx/kernel.in # -r requirements/edx/paver.txt @@ -1207,11 +1208,11 @@ testfixtures==8.3.0 # via edx-enterprise text-unidecode==1.3 # via python-slugify -tinycss2==1.2.1 +tinycss2==1.4.0 # via bleach tomlkit==0.13.2 # via snowflake-connector-python -tqdm==4.66.6 +tqdm==4.67.1 # via # nltk # openai @@ -1257,7 +1258,7 @@ voluptuous==0.15.2 # via ora2 walrus==0.9.4 # via edx-event-bus-redis -watchdog==5.0.3 +watchdog==6.0.0 # via -r requirements/edx/paver.txt wcwidth==0.2.13 # via prompt-toolkit @@ -1278,7 +1279,7 @@ webob==1.8.9 # via # -r requirements/edx/kernel.in # xblock -wrapt==1.16.0 +wrapt==1.17.0 # via -r requirements/edx/paver.txt xblock[django]==5.1.0 # via @@ -1312,9 +1313,9 @@ xmlsec==1.3.14 # via python3-saml xss-utils==0.6.0 # via -r requirements/edx/kernel.in -yarl==1.17.0 +yarl==1.18.3 # via aiohttp -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/edx/coverage.txt b/requirements/edx/coverage.txt index 3635567f8906..38acef7c7978 100644 --- a/requirements/edx/coverage.txt +++ b/requirements/edx/coverage.txt @@ -6,7 +6,7 @@ # chardet==5.2.0 # via diff-cover -coverage==7.6.4 +coverage==7.6.8 # via -r requirements/edx/coverage.in diff-cover==9.2.0 # via -r requirements/edx/coverage.in diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 5fe6f26fc0ca..08484563c110 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -16,12 +16,12 @@ acid-xblock==0.4.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # aiohttp -aiohttp==3.10.10 +aiohttp==3.11.9 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -41,7 +41,7 @@ algoliasearch==3.0.0 # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -amqp==5.2.0 +amqp==5.3.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -60,7 +60,7 @@ annotated-types==0.7.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # -r requirements/edx/testing.txt # starlette @@ -113,7 +113,7 @@ backoff==1.10.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # analytics-python -bcrypt==4.2.0 +bcrypt==4.2.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -130,7 +130,7 @@ billiard==4.2.1 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # celery -bleach[css]==6.1.0 +bleach[css]==6.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -144,14 +144,14 @@ boto==2.49.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -boto3==1.35.50 +boto3==1.35.76 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # django-ses # fs-s3fs # ora2 -botocore==1.35.50 +botocore==1.35.76 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -165,7 +165,7 @@ build==1.2.2.post1 # via # -r requirements/edx/../pip-tools.txt # pip-tools -cachecontrol==0.14.0 +cachecontrol==0.14.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -267,7 +267,7 @@ click-repl==0.3.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # celery -code-annotations==1.8.0 +code-annotations==2.0.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -282,7 +282,7 @@ colorama==0.4.6 # via # -r requirements/edx/testing.txt # tox -coverage[toml]==7.6.4 +coverage[toml]==7.6.8 # via # -r requirements/edx/testing.txt # pytest-cov @@ -290,7 +290,7 @@ crowdsourcehinter-xblock==0.8 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -cryptography==43.0.3 +cryptography==44.0.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -337,7 +337,7 @@ distlib==0.3.9 # via # -r requirements/edx/testing.txt # virtualenv -django==4.2.16 +django==4.2.17 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -440,7 +440,7 @@ django-config-models==2.7.0 # edx-enterprise # edx-name-affirmation # lti-consumer-xblock -django-cors-headers==4.5.0 +django-cors-headers==4.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -516,7 +516,7 @@ django-multi-email-field==0.7.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-enterprise -django-mysql==4.14.0 +django-mysql==4.15.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -549,7 +549,7 @@ django-sekizai==4.1.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-django-wiki -django-ses==4.2.0 +django-ses==4.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -563,7 +563,7 @@ django-simple-history==3.4.0 # edx-organizations # edx-proctoring # ora2 -django-statici18n==2.5.0 +django-statici18n==2.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -588,7 +588,7 @@ django-user-tasks==3.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -django-waffle==4.1.0 +django-waffle==4.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -654,7 +654,7 @@ drf-jwt==1.19.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-drf-extensions -drf-spectacular==0.27.2 +drf-spectacular==0.28.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -687,7 +687,7 @@ edx-bulk-grades==1.1.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # staff-graded-xblock -edx-ccx-keys==1.3.0 +edx-ccx-keys==2.0.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -703,7 +703,7 @@ edx-codejail==3.5.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-completion==4.7.3 +edx-completion==4.7.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -717,7 +717,7 @@ edx-django-sites-extensions==4.2.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-django-utils==7.0.0 +edx-django-utils==7.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -748,7 +748,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.3.1 +edx-enterprise==5.4.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/doc.txt @@ -798,7 +798,7 @@ edx-organizations==6.13.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-proctoring==4.18.3 +edx-proctoring==4.18.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -823,7 +823,7 @@ edx-sga==0.25.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-submissions==3.8.2 +edx-submissions==3.8.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -855,7 +855,7 @@ edx-when==2.5.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-proctoring -edxval==2.6.0 +edxval==2.6.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -889,11 +889,11 @@ execnet==2.1.1 # pytest-xdist factory-boy==3.3.1 # via -r requirements/edx/testing.txt -faker==30.8.1 +faker==33.1.0 # via # -r requirements/edx/testing.txt # factory-boy -fastapi==0.115.4 +fastapi==0.115.6 # via # -r requirements/edx/testing.txt # pact-python @@ -909,7 +909,7 @@ filelock==3.16.1 # snowflake-connector-python # tox # virtualenv -firebase-admin==6.5.0 +firebase-admin==6.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -939,7 +939,7 @@ future==1.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # pyjwkest -geoip2==4.8.0 +geoip2==4.8.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -953,7 +953,7 @@ glob2==0.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -google-api-core[grpc]==2.22.0 +google-api-core[grpc]==2.23.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -962,12 +962,12 @@ google-api-core[grpc]==2.22.0 # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.149.0 +google-api-python-client==2.154.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # firebase-admin -google-auth==2.35.0 +google-auth==2.36.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -993,7 +993,7 @@ google-cloud-firestore==2.19.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # firebase-admin -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1009,7 +1009,7 @@ google-resumable-media==2.7.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # google-cloud-storage -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1019,13 +1019,13 @@ grimp==3.5 # via # -r requirements/edx/testing.txt # import-linter -grpcio==1.67.0 +grpcio==1.68.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # google-api-core # grpcio-status -grpcio-status==1.67.0 +grpcio-status==1.68.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1055,7 +1055,7 @@ httplib2==0.22.0 # google-auth-httplib2 httpretty==1.1.4 # via -r requirements/edx/testing.txt -icalendar==6.0.1 +icalendar==6.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1191,7 +1191,7 @@ loremipsum==1.0.5 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # ora2 -lti-consumer-xblock==9.11.3 +lti-consumer-xblock==9.12.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1210,7 +1210,7 @@ lxml[html-clean]==5.3.0 # python3-saml # xblock # xmlsec -lxml-html-clean==0.3.1 +lxml-html-clean==0.4.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1219,7 +1219,7 @@ mailsnake==1.6.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -mako==1.3.6 +mako==1.3.7 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1253,7 +1253,7 @@ mccabe==0.7.0 # via # -r requirements/edx/testing.txt # pylint -meilisearch==0.31.6 +meilisearch==0.33.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1305,17 +1305,17 @@ mypy==1.11.2 # djangorestframework-stubs mypy-extensions==1.0.0 # via mypy -mysqlclient==2.2.5 +mysqlclient==2.2.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-forum -newrelic==10.2.0 +newrelic==10.3.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-django-utils -nh3==0.2.18 +nh3==0.2.19 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1361,7 +1361,7 @@ openedx-atlas==0.6.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # openedx-forum -openedx-calc==3.1.2 +openedx-calc==4.0.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1396,7 +1396,7 @@ openedx-filters==1.11.0 # -r requirements/edx/testing.txt # lti-consumer-xblock # ora2 -openedx-forum==0.1.3 +openedx-forum==0.1.5 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1418,7 +1418,7 @@ ora2==6.14.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -packaging==24.1 +packaging==24.2 # via # -r requirements/edx/../pip-tools.txt # -r requirements/edx/doc.txt @@ -1434,7 +1434,7 @@ packaging==24.1 # tox pact-python==2.2.2 # via -r requirements/edx/testing.txt -pansi==2020.7.3 +pansi==2024.11.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1487,6 +1487,7 @@ pillow==11.0.0 # edx-enterprise # edx-organizations # edxval + # pansi pip-tools==7.4.1 # via -r requirements/edx/../pip-tools.txt platformdirs==4.3.6 @@ -1513,10 +1514,11 @@ prompt-toolkit==3.0.48 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # click-repl -propcache==0.2.0 +propcache==0.2.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt + # aiohttp # yarl proto-plus==1.25.0 # via @@ -1524,7 +1526,7 @@ proto-plus==1.25.0 # -r requirements/edx/testing.txt # google-api-core # google-cloud-firestore -protobuf==5.28.3 +protobuf==5.29.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1579,13 +1581,13 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.9.2 +pydantic==2.10.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # camel-converter # fastapi -pydantic-core==2.23.4 +pydantic-core==2.27.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1610,7 +1612,7 @@ pyjwkest==1.4.2 # -r requirements/edx/testing.txt # edx-token-utils # lti-consumer-xblock -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1680,7 +1682,7 @@ pynliner==0.8.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1714,7 +1716,7 @@ pysrt==1.1.2 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edxval -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/edx/testing.txt # pylint-pytest @@ -1727,7 +1729,7 @@ pytest==8.3.3 # pytest-xdist pytest-attrib==0.1.3 # via -r requirements/edx/testing.txt -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/edx/testing.txt pytest-django==4.9.0 # via -r requirements/edx/testing.txt @@ -1841,7 +1843,7 @@ referencing==0.35.1 # -r requirements/edx/testing.txt # jsonschema # jsonschema-specifications -regex==2024.9.11 +regex==2024.11.6 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1883,7 +1885,7 @@ requests-oauthlib==2.0.0 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # social-auth-core -rpds-py==0.20.0 +rpds-py==0.22.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1901,7 +1903,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.10.3 +s3transfer==0.10.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -1936,13 +1938,12 @@ simplejson==3.19.3 # xblock-utils singledispatch==4.1.0 # via -r requirements/edx/testing.txt -six==1.16.0 +six==1.17.0 # via # -r requirements/edx/assets.txt # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # analytics-python - # bleach # codejail-includes # crowdsourcehinter-xblock # edx-ace @@ -1961,7 +1962,6 @@ six==1.16.0 # libsass # optimizely-sdk # pact-python - # pansi # paver # py2neo # pyjwkest @@ -1985,7 +1985,7 @@ snowballstemmer==2.2.0 # via # -r requirements/edx/doc.txt # sphinx -snowflake-connector-python==3.12.3 +snowflake-connector-python==3.12.4 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2029,7 +2029,7 @@ sphinx==8.1.3 # sphinxcontrib-httpdomain # sphinxcontrib-openapi # sphinxext-rediraffe -sphinx-autoapi==3.3.3 +sphinx-autoapi==3.4.0 # via -r requirements/edx/doc.txt sphinx-book-theme==1.1.3 # via -r requirements/edx/doc.txt @@ -2073,7 +2073,7 @@ sphinxcontrib-serializinghtml==2.0.0 # sphinx sphinxext-rediraffe==0.2.7 # via -r requirements/edx/doc.txt -sqlparse==0.5.1 +sqlparse==0.5.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2083,11 +2083,11 @@ staff-graded-xblock==2.3.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -starlette==0.41.2 +starlette==0.41.3 # via # -r requirements/edx/testing.txt # fastapi -stevedore==5.3.0 +stevedore==5.4.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2116,12 +2116,12 @@ text-unidecode==1.3 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # python-slugify -tinycss2==1.2.1 +tinycss2==1.4.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # bleach -tomli==2.0.2 +tomli==2.2.1 # via django-stubs tomlkit==0.13.2 # via @@ -2131,7 +2131,7 @@ tomlkit==0.13.2 # snowflake-connector-python tox==4.23.2 # via -r requirements/edx/testing.txt -tqdm==4.66.6 +tqdm==4.67.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2149,6 +2149,7 @@ typing-extensions==4.12.2 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt + # anyio # django-countries # django-stubs # django-stubs-ext @@ -2199,7 +2200,7 @@ user-util==1.1.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -uvicorn==0.32.0 +uvicorn==0.32.1 # via # -r requirements/edx/testing.txt # pact-python @@ -2210,7 +2211,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.27.1 +virtualenv==20.28.0 # via # -r requirements/edx/testing.txt # tox @@ -2226,7 +2227,7 @@ walrus==0.9.4 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # edx-event-bus-redis -watchdog==5.0.3 +watchdog==6.0.0 # via # -r requirements/edx/development.in # -r requirements/edx/doc.txt @@ -2257,11 +2258,11 @@ webob==1.8.9 # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # xblock -wheel==0.44.0 +wheel==0.45.1 # via # -r requirements/edx/../pip-tools.txt # pip-tools -wrapt==1.16.0 +wrapt==1.17.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt @@ -2314,13 +2315,13 @@ xss-utils==0.6.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -yarl==1.17.0 +yarl==1.18.3 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt # aiohttp # pact-python -zipp==3.20.2 +zipp==3.21.0 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index cc0bc64a6e70..ec4ada9eaae3 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -10,11 +10,11 @@ accessible-pygments==0.0.5 # via pydata-sphinx-theme acid-xblock==0.4.1 # via -r requirements/edx/base.txt -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via # -r requirements/edx/base.txt # aiohttp -aiohttp==3.10.10 +aiohttp==3.11.9 # via # -r requirements/edx/base.txt # geoip2 @@ -29,7 +29,7 @@ algoliasearch==3.0.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -amqp==5.2.0 +amqp==5.3.1 # via # -r requirements/edx/base.txt # kombu @@ -82,7 +82,7 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -bcrypt==4.2.0 +bcrypt==4.2.1 # via # -r requirements/edx/base.txt # paramiko @@ -96,7 +96,7 @@ billiard==4.2.1 # via # -r requirements/edx/base.txt # celery -bleach[css]==6.1.0 +bleach[css]==6.2.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -107,20 +107,20 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.35.50 +boto3==1.35.76 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.35.50 +botocore==1.35.76 # via # -r requirements/edx/base.txt # boto3 # s3transfer bridgekeeper==0.9 # via -r requirements/edx/base.txt -cachecontrol==0.14.0 +cachecontrol==0.14.1 # via # -r requirements/edx/base.txt # firebase-admin @@ -191,7 +191,7 @@ click-repl==0.3.0 # via # -r requirements/edx/base.txt # celery -code-annotations==1.8.0 +code-annotations==2.0.0 # via # -r requirements/edx/base.txt # -r requirements/edx/doc.in @@ -201,7 +201,7 @@ codejail-includes==1.0.0 # via -r requirements/edx/base.txt crowdsourcehinter-xblock==0.8 # via -r requirements/edx/base.txt -cryptography==43.0.3 +cryptography==44.0.0 # via # -r requirements/edx/base.txt # django-fernet-fields-v2 @@ -227,7 +227,7 @@ defusedxml==0.7.1 # ora2 # python3-openid # social-auth-core -django==4.2.16 +django==4.2.17 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -320,7 +320,7 @@ django-config-models==2.7.0 # edx-enterprise # edx-name-affirmation # lti-consumer-xblock -django-cors-headers==4.5.0 +django-cors-headers==4.6.0 # via -r requirements/edx/base.txt django-countries==7.6.1 # via @@ -381,7 +381,7 @@ django-multi-email-field==0.7.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-mysql==4.14.0 +django-mysql==4.15.0 # via -r requirements/edx/base.txt django-oauth-toolkit==1.7.1 # via @@ -404,7 +404,7 @@ django-sekizai==4.1.0 # via # -r requirements/edx/base.txt # openedx-django-wiki -django-ses==4.2.0 +django-ses==4.3.0 # via -r requirements/edx/base.txt django-simple-history==3.4.0 # via @@ -415,7 +415,7 @@ django-simple-history==3.4.0 # edx-organizations # edx-proctoring # ora2 -django-statici18n==2.5.0 +django-statici18n==2.6.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock @@ -429,7 +429,7 @@ django-storages==1.14.3 # edxval django-user-tasks==3.2.0 # via -r requirements/edx/base.txt -django-waffle==4.1.0 +django-waffle==4.2.0 # via # -r requirements/edx/base.txt # edx-django-utils @@ -482,7 +482,7 @@ drf-jwt==1.19.2 # via # -r requirements/edx/base.txt # edx-drf-extensions -drf-spectacular==0.27.2 +drf-spectacular==0.28.0 # via -r requirements/edx/base.txt drf-yasg==1.21.8 # via @@ -505,7 +505,7 @@ edx-bulk-grades==1.1.0 # via # -r requirements/edx/base.txt # staff-graded-xblock -edx-ccx-keys==1.3.0 +edx-ccx-keys==2.0.2 # via # -r requirements/edx/base.txt # lti-consumer-xblock @@ -517,7 +517,7 @@ edx-celeryutils==1.3.0 # super-csv edx-codejail==3.5.2 # via -r requirements/edx/base.txt -edx-completion==4.7.3 +edx-completion==4.7.6 # via -r requirements/edx/base.txt edx-django-release-util==1.4.0 # via @@ -526,7 +526,7 @@ edx-django-release-util==1.4.0 # edxval edx-django-sites-extensions==4.2.0 # via -r requirements/edx/base.txt -edx-django-utils==7.0.0 +edx-django-utils==7.1.0 # via # -r requirements/edx/base.txt # django-config-models @@ -555,7 +555,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.3.1 +edx-enterprise==5.4.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -590,7 +590,7 @@ edx-opaque-keys[django]==2.11.0 # ora2 edx-organizations==6.13.0 # via -r requirements/edx/base.txt -edx-proctoring==4.18.3 +edx-proctoring==4.18.4 # via # -r requirements/edx/base.txt # edx-proctoring-proctortrack @@ -609,7 +609,7 @@ edx-search==4.1.1 # openedx-forum edx-sga==0.25.0 # via -r requirements/edx/base.txt -edx-submissions==3.8.2 +edx-submissions==3.8.3 # via # -r requirements/edx/base.txt # ora2 @@ -635,7 +635,7 @@ edx-when==2.5.0 # via # -r requirements/edx/base.txt # edx-proctoring -edxval==2.6.0 +edxval==2.6.1 # via -r requirements/edx/base.txt elasticsearch==7.9.1 # via @@ -664,7 +664,7 @@ filelock==3.16.1 # via # -r requirements/edx/base.txt # snowflake-connector-python -firebase-admin==6.5.0 +firebase-admin==6.6.0 # via # -r requirements/edx/base.txt # edx-ace @@ -687,7 +687,7 @@ future==1.0.0 # via # -r requirements/edx/base.txt # pyjwkest -geoip2==4.8.0 +geoip2==4.8.1 # via -r requirements/edx/base.txt gitdb==4.0.11 # via gitpython @@ -695,7 +695,7 @@ gitpython==3.1.43 # via -r requirements/edx/doc.in glob2==0.7 # via -r requirements/edx/base.txt -google-api-core[grpc]==2.22.0 +google-api-core[grpc]==2.23.0 # via # -r requirements/edx/base.txt # firebase-admin @@ -703,11 +703,11 @@ google-api-core[grpc]==2.22.0 # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.149.0 +google-api-python-client==2.154.0 # via # -r requirements/edx/base.txt # firebase-admin -google-auth==2.35.0 +google-auth==2.36.0 # via # -r requirements/edx/base.txt # google-api-core @@ -729,7 +729,7 @@ google-cloud-firestore==2.19.0 # via # -r requirements/edx/base.txt # firebase-admin -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via # -r requirements/edx/base.txt # firebase-admin @@ -742,17 +742,17 @@ google-resumable-media==2.7.2 # via # -r requirements/edx/base.txt # google-cloud-storage -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio==1.67.0 +grpcio==1.68.1 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio-status==1.67.0 +grpcio-status==1.68.1 # via # -r requirements/edx/base.txt # google-api-core @@ -769,7 +769,7 @@ httplib2==0.22.0 # -r requirements/edx/base.txt # google-api-python-client # google-auth-httplib2 -icalendar==6.0.1 +icalendar==6.1.0 # via -r requirements/edx/base.txt idna==3.10 # via @@ -863,7 +863,7 @@ loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==9.11.3 +lti-consumer-xblock==9.12.0 # via -r requirements/edx/base.txt lxml[html-clean]==5.3.0 # via @@ -878,13 +878,13 @@ lxml[html-clean]==5.3.0 # python3-saml # xblock # xmlsec -lxml-html-clean==0.3.1 +lxml-html-clean==0.4.1 # via # -r requirements/edx/base.txt # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt -mako==1.3.6 +mako==1.3.7 # via # -r requirements/edx/base.txt # acid-xblock @@ -910,7 +910,7 @@ maxminddb==2.6.2 # via # -r requirements/edx/base.txt # geoip2 -meilisearch==0.31.6 +meilisearch==0.33.0 # via # -r requirements/edx/base.txt # edx-search @@ -942,15 +942,15 @@ multidict==6.1.0 # -r requirements/edx/base.txt # aiohttp # yarl -mysqlclient==2.2.5 +mysqlclient==2.2.6 # via # -r requirements/edx/base.txt # openedx-forum -newrelic==10.2.0 +newrelic==10.3.1 # via # -r requirements/edx/base.txt # edx-django-utils -nh3==0.2.18 +nh3==0.2.19 # via -r requirements/edx/base.txt nltk==3.9.1 # via @@ -984,7 +984,7 @@ openedx-atlas==0.6.2 # via # -r requirements/edx/base.txt # openedx-forum -openedx-calc==3.1.2 +openedx-calc==4.0.1 # via -r requirements/edx/base.txt openedx-django-pyfs==3.7.0 # via @@ -1010,7 +1010,7 @@ openedx-filters==1.11.0 # -r requirements/edx/base.txt # lti-consumer-xblock # ora2 -openedx-forum==0.1.3 +openedx-forum==0.1.5 # via -r requirements/edx/base.txt openedx-learning==0.18.1 # via @@ -1024,7 +1024,7 @@ optimizely-sdk==4.1.1 # -r requirements/edx/base.txt ora2==6.14.1 # via -r requirements/edx/base.txt -packaging==24.1 +packaging==24.2 # via # -r requirements/edx/base.txt # drf-yasg @@ -1032,7 +1032,7 @@ packaging==24.1 # py2neo # snowflake-connector-python # sphinx -pansi==2020.7.3 +pansi==2024.11.0 # via # -r requirements/edx/base.txt # py2neo @@ -1072,6 +1072,7 @@ pillow==11.0.0 # edx-enterprise # edx-organizations # edxval + # pansi platformdirs==4.3.6 # via # -r requirements/edx/base.txt @@ -1084,16 +1085,17 @@ prompt-toolkit==3.0.48 # via # -r requirements/edx/base.txt # click-repl -propcache==0.2.0 +propcache==0.2.1 # via # -r requirements/edx/base.txt + # aiohttp # yarl proto-plus==1.25.0 # via # -r requirements/edx/base.txt # google-api-core # google-cloud-firestore -protobuf==5.28.3 +protobuf==5.29.1 # via # -r requirements/edx/base.txt # google-api-core @@ -1131,11 +1133,11 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.9.2 +pydantic==2.10.3 # via # -r requirements/edx/base.txt # camel-converter -pydantic-core==2.23.4 +pydantic-core==2.27.1 # via # -r requirements/edx/base.txt # pydantic @@ -1154,7 +1156,7 @@ pyjwkest==1.4.2 # -r requirements/edx/base.txt # edx-token-utils # lti-consumer-xblock -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.1 # via # -r requirements/edx/base.txt # drf-jwt @@ -1190,7 +1192,7 @@ pynacl==1.5.0 # paramiko pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # -r requirements/edx/base.txt # optimizely-sdk @@ -1286,7 +1288,7 @@ referencing==0.35.1 # -r requirements/edx/base.txt # jsonschema # jsonschema-specifications -regex==2024.9.11 +regex==2024.11.6 # via # -r requirements/edx/base.txt # nltk @@ -1323,7 +1325,7 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/base.txt # social-auth-core -rpds-py==0.20.0 +rpds-py==0.22.3 # via # -r requirements/edx/base.txt # jsonschema @@ -1338,7 +1340,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.10.3 +s3transfer==0.10.4 # via # -r requirements/edx/base.txt # boto3 @@ -1364,11 +1366,10 @@ simplejson==3.19.3 # super-csv # xblock # xblock-utils -six==1.16.0 +six==1.17.0 # via # -r requirements/edx/base.txt # analytics-python - # bleach # codejail-includes # crowdsourcehinter-xblock # edx-ace @@ -1385,7 +1386,6 @@ six==1.16.0 # interchange # libsass # optimizely-sdk - # pansi # paver # py2neo # pyjwkest @@ -1400,7 +1400,7 @@ smmap==5.0.1 # via gitdb snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python==3.12.3 +snowflake-connector-python==3.12.4 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1438,7 +1438,7 @@ sphinx==8.1.3 # sphinxcontrib-httpdomain # sphinxcontrib-openapi # sphinxext-rediraffe -sphinx-autoapi==3.3.3 +sphinx-autoapi==3.4.0 # via -r requirements/edx/doc.in sphinx-book-theme==1.1.3 # via -r requirements/edx/doc.in @@ -1466,13 +1466,13 @@ sphinxcontrib-serializinghtml==2.0.0 # via sphinx sphinxext-rediraffe==0.2.7 # via -r requirements/edx/doc.in -sqlparse==0.5.1 +sqlparse==0.5.2 # via # -r requirements/edx/base.txt # django staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt -stevedore==5.3.0 +stevedore==5.4.0 # via # -r requirements/edx/base.txt # code-annotations @@ -1496,7 +1496,7 @@ text-unidecode==1.3 # via # -r requirements/edx/base.txt # python-slugify -tinycss2==1.2.1 +tinycss2==1.4.0 # via # -r requirements/edx/base.txt # bleach @@ -1504,7 +1504,7 @@ tomlkit==0.13.2 # via # -r requirements/edx/base.txt # snowflake-connector-python -tqdm==4.66.6 +tqdm==4.67.1 # via # -r requirements/edx/base.txt # nltk @@ -1559,7 +1559,7 @@ walrus==0.9.4 # via # -r requirements/edx/base.txt # edx-event-bus-redis -watchdog==5.0.3 +watchdog==6.0.0 # via -r requirements/edx/base.txt wcwidth==0.2.13 # via @@ -1583,7 +1583,7 @@ webob==1.8.9 # via # -r requirements/edx/base.txt # xblock -wrapt==1.16.0 +wrapt==1.17.0 # via # -r requirements/edx/base.txt # astroid @@ -1622,11 +1622,11 @@ xmlsec==1.3.14 # python3-saml xss-utils==0.6.0 # via -r requirements/edx/base.txt -yarl==1.17.0 +yarl==1.18.3 # via # -r requirements/edx/base.txt # aiohttp -zipp==3.20.2 +zipp==3.21.0 # via # -r requirements/edx/base.txt # importlib-metadata diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index f3dae3b0efda..c9ee8f3aff49 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -47,11 +47,11 @@ python-memcached==1.62 # via -r requirements/edx/paver.in requests==2.32.3 # via -r requirements/edx/paver.in -six==1.16.0 +six==1.17.0 # via # libsass # paver -stevedore==5.3.0 +stevedore==5.4.0 # via # -r requirements/edx/paver.in # edx-opaque-keys @@ -59,7 +59,7 @@ typing-extensions==4.12.2 # via edx-opaque-keys urllib3==2.2.3 # via requests -watchdog==5.0.3 +watchdog==6.0.0 # via -r requirements/edx/paver.in -wrapt==1.16.0 +wrapt==1.17.0 # via -r requirements/edx/paver.in diff --git a/requirements/edx/semgrep.txt b/requirements/edx/semgrep.txt index 61db06bbf123..c244159342bc 100644 --- a/requirements/edx/semgrep.txt +++ b/requirements/edx/semgrep.txt @@ -34,17 +34,17 @@ colorama==0.4.6 # via semgrep defusedxml==0.7.1 # via semgrep -deprecated==1.2.14 +deprecated==1.2.15 # via # opentelemetry-api # opentelemetry-exporter-otlp-proto-http exceptiongroup==1.2.2 # via semgrep -face==22.0.0 +face==24.0.0 # via glom glom==22.1.0 # via semgrep -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via opentelemetry-exporter-otlp-proto-http idna==3.10 # via requests @@ -88,9 +88,9 @@ opentelemetry-semantic-conventions==0.46b0 # opentelemetry-sdk opentelemetry-util-http==0.46b0 # via opentelemetry-instrumentation-requests -packaging==24.1 +packaging==24.2 # via semgrep -peewee==3.17.7 +peewee==3.17.8 # via semgrep protobuf==4.25.5 # via @@ -108,7 +108,7 @@ requests==2.32.3 # semgrep rich==13.5.3 # via semgrep -rpds-py==0.20.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -116,7 +116,7 @@ ruamel-yaml==0.17.40 # via semgrep ruamel-yaml-clib==0.2.12 # via ruamel-yaml -semgrep==1.93.0 +semgrep==1.97.0 # via -r requirements/edx/semgrep.in tomli==2.0.2 # via semgrep @@ -130,11 +130,11 @@ urllib3==2.2.3 # semgrep wcmatch==8.5.2 # via semgrep -wrapt==1.16.0 +wrapt==1.17.0 # via # deprecated # opentelemetry-instrumentation -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 9181ab448aee..6edea84c69bf 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -8,11 +8,11 @@ # via -r requirements/edx/base.txt acid-xblock==0.4.1 # via -r requirements/edx/base.txt -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via # -r requirements/edx/base.txt # aiohttp -aiohttp==3.10.10 +aiohttp==3.11.9 # via # -r requirements/edx/base.txt # geoip2 @@ -25,7 +25,7 @@ algoliasearch==3.0.0 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt -amqp==5.2.0 +amqp==5.3.1 # via # -r requirements/edx/base.txt # kombu @@ -39,7 +39,7 @@ annotated-types==0.7.0 # via # -r requirements/edx/base.txt # pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via starlette appdirs==1.4.4 # via @@ -79,7 +79,7 @@ backoff==1.10.0 # via # -r requirements/edx/base.txt # analytics-python -bcrypt==4.2.0 +bcrypt==4.2.1 # via # -r requirements/edx/base.txt # paramiko @@ -93,7 +93,7 @@ billiard==4.2.1 # via # -r requirements/edx/base.txt # celery -bleach[css]==6.1.0 +bleach[css]==6.2.0 # via # -r requirements/edx/base.txt # edx-enterprise @@ -104,20 +104,20 @@ bleach[css]==6.1.0 # xblock-poll boto==2.49.0 # via -r requirements/edx/base.txt -boto3==1.35.50 +boto3==1.35.76 # via # -r requirements/edx/base.txt # django-ses # fs-s3fs # ora2 -botocore==1.35.50 +botocore==1.35.76 # via # -r requirements/edx/base.txt # boto3 # s3transfer bridgekeeper==0.9 # via -r requirements/edx/base.txt -cachecontrol==0.14.0 +cachecontrol==0.14.1 # via # -r requirements/edx/base.txt # firebase-admin @@ -200,7 +200,7 @@ click-repl==0.3.0 # via # -r requirements/edx/base.txt # celery -code-annotations==1.8.0 +code-annotations==2.0.0 # via # -r requirements/edx/base.txt # -r requirements/edx/testing.in @@ -211,13 +211,13 @@ codejail-includes==1.0.0 # via -r requirements/edx/base.txt colorama==0.4.6 # via tox -coverage[toml]==7.6.4 +coverage[toml]==7.6.8 # via # -r requirements/edx/coverage.txt # pytest-cov crowdsourcehinter-xblock==0.8 # via -r requirements/edx/base.txt -cryptography==43.0.3 +cryptography==44.0.0 # via # -r requirements/edx/base.txt # django-fernet-fields-v2 @@ -253,7 +253,7 @@ dill==0.3.9 # via pylint distlib==0.3.9 # via virtualenv -django==4.2.16 +django==4.2.17 # via # -c requirements/edx/../common_constraints.txt # -c requirements/edx/../constraints.txt @@ -346,7 +346,7 @@ django-config-models==2.7.0 # edx-enterprise # edx-name-affirmation # lti-consumer-xblock -django-cors-headers==4.5.0 +django-cors-headers==4.6.0 # via -r requirements/edx/base.txt django-countries==7.6.1 # via @@ -407,7 +407,7 @@ django-multi-email-field==0.7.0 # via # -r requirements/edx/base.txt # edx-enterprise -django-mysql==4.14.0 +django-mysql==4.15.0 # via -r requirements/edx/base.txt django-oauth-toolkit==1.7.1 # via @@ -430,7 +430,7 @@ django-sekizai==4.1.0 # via # -r requirements/edx/base.txt # openedx-django-wiki -django-ses==4.2.0 +django-ses==4.3.0 # via -r requirements/edx/base.txt django-simple-history==3.4.0 # via @@ -441,7 +441,7 @@ django-simple-history==3.4.0 # edx-organizations # edx-proctoring # ora2 -django-statici18n==2.5.0 +django-statici18n==2.6.0 # via # -r requirements/edx/base.txt # lti-consumer-xblock @@ -455,7 +455,7 @@ django-storages==1.14.3 # edxval django-user-tasks==3.2.0 # via -r requirements/edx/base.txt -django-waffle==4.1.0 +django-waffle==4.2.0 # via # -r requirements/edx/base.txt # edx-django-utils @@ -503,7 +503,7 @@ drf-jwt==1.19.2 # via # -r requirements/edx/base.txt # edx-drf-extensions -drf-spectacular==0.27.2 +drf-spectacular==0.28.0 # via -r requirements/edx/base.txt drf-yasg==1.21.8 # via @@ -526,7 +526,7 @@ edx-bulk-grades==1.1.0 # via # -r requirements/edx/base.txt # staff-graded-xblock -edx-ccx-keys==1.3.0 +edx-ccx-keys==2.0.2 # via # -r requirements/edx/base.txt # lti-consumer-xblock @@ -538,7 +538,7 @@ edx-celeryutils==1.3.0 # super-csv edx-codejail==3.5.2 # via -r requirements/edx/base.txt -edx-completion==4.7.3 +edx-completion==4.7.6 # via -r requirements/edx/base.txt edx-django-release-util==1.4.0 # via @@ -547,7 +547,7 @@ edx-django-release-util==1.4.0 # edxval edx-django-sites-extensions==4.2.0 # via -r requirements/edx/base.txt -edx-django-utils==7.0.0 +edx-django-utils==7.1.0 # via # -r requirements/edx/base.txt # django-config-models @@ -576,7 +576,7 @@ edx-drf-extensions==10.5.0 # edx-when # edxval # openedx-learning -edx-enterprise==5.3.1 +edx-enterprise==5.4.2 # via # -c requirements/edx/../constraints.txt # -r requirements/edx/base.txt @@ -613,7 +613,7 @@ edx-opaque-keys[django]==2.11.0 # ora2 edx-organizations==6.13.0 # via -r requirements/edx/base.txt -edx-proctoring==4.18.3 +edx-proctoring==4.18.4 # via # -r requirements/edx/base.txt # edx-proctoring-proctortrack @@ -632,7 +632,7 @@ edx-search==4.1.1 # openedx-forum edx-sga==0.25.0 # via -r requirements/edx/base.txt -edx-submissions==3.8.2 +edx-submissions==3.8.3 # via # -r requirements/edx/base.txt # ora2 @@ -658,7 +658,7 @@ edx-when==2.5.0 # via # -r requirements/edx/base.txt # edx-proctoring -edxval==2.6.0 +edxval==2.6.1 # via -r requirements/edx/base.txt elasticsearch==7.9.1 # via @@ -683,9 +683,9 @@ execnet==2.1.1 # via pytest-xdist factory-boy==3.3.1 # via -r requirements/edx/testing.in -faker==30.8.1 +faker==33.1.0 # via factory-boy -fastapi==0.115.4 +fastapi==0.115.6 # via pact-python fastavro==1.9.7 # via @@ -697,7 +697,7 @@ filelock==3.16.1 # snowflake-connector-python # tox # virtualenv -firebase-admin==6.5.0 +firebase-admin==6.6.0 # via # -r requirements/edx/base.txt # edx-ace @@ -722,11 +722,11 @@ future==1.0.0 # via # -r requirements/edx/base.txt # pyjwkest -geoip2==4.8.0 +geoip2==4.8.1 # via -r requirements/edx/base.txt glob2==0.7 # via -r requirements/edx/base.txt -google-api-core[grpc]==2.22.0 +google-api-core[grpc]==2.23.0 # via # -r requirements/edx/base.txt # firebase-admin @@ -734,11 +734,11 @@ google-api-core[grpc]==2.22.0 # google-cloud-core # google-cloud-firestore # google-cloud-storage -google-api-python-client==2.149.0 +google-api-python-client==2.154.0 # via # -r requirements/edx/base.txt # firebase-admin -google-auth==2.35.0 +google-auth==2.36.0 # via # -r requirements/edx/base.txt # google-api-core @@ -760,7 +760,7 @@ google-cloud-firestore==2.19.0 # via # -r requirements/edx/base.txt # firebase-admin -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via # -r requirements/edx/base.txt # firebase-admin @@ -773,19 +773,19 @@ google-resumable-media==2.7.2 # via # -r requirements/edx/base.txt # google-cloud-storage -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status grimp==3.5 # via import-linter -grpcio==1.67.0 +grpcio==1.68.1 # via # -r requirements/edx/base.txt # google-api-core # grpcio-status -grpcio-status==1.67.0 +grpcio-status==1.68.1 # via # -r requirements/edx/base.txt # google-api-core @@ -806,7 +806,7 @@ httplib2==0.22.0 # google-auth-httplib2 httpretty==1.1.4 # via -r requirements/edx/testing.in -icalendar==6.0.1 +icalendar==6.1.0 # via -r requirements/edx/base.txt idna==3.10 # via @@ -906,7 +906,7 @@ loremipsum==1.0.5 # via # -r requirements/edx/base.txt # ora2 -lti-consumer-xblock==9.11.3 +lti-consumer-xblock==9.12.0 # via -r requirements/edx/base.txt lxml[html-clean]==5.3.0 # via @@ -922,13 +922,13 @@ lxml[html-clean]==5.3.0 # python3-saml # xblock # xmlsec -lxml-html-clean==0.3.1 +lxml-html-clean==0.4.1 # via # -r requirements/edx/base.txt # lxml mailsnake==1.6.4 # via -r requirements/edx/base.txt -mako==1.3.6 +mako==1.3.7 # via # -r requirements/edx/base.txt # acid-xblock @@ -957,7 +957,7 @@ maxminddb==2.6.2 # geoip2 mccabe==0.7.0 # via pylint -meilisearch==0.31.6 +meilisearch==0.33.0 # via # -r requirements/edx/base.txt # edx-search @@ -987,15 +987,15 @@ multidict==6.1.0 # -r requirements/edx/base.txt # aiohttp # yarl -mysqlclient==2.2.5 +mysqlclient==2.2.6 # via # -r requirements/edx/base.txt # openedx-forum -newrelic==10.2.0 +newrelic==10.3.1 # via # -r requirements/edx/base.txt # edx-django-utils -nh3==0.2.18 +nh3==0.2.19 # via -r requirements/edx/base.txt nltk==3.9.1 # via @@ -1029,7 +1029,7 @@ openedx-atlas==0.6.2 # via # -r requirements/edx/base.txt # openedx-forum -openedx-calc==3.1.2 +openedx-calc==4.0.1 # via -r requirements/edx/base.txt openedx-django-pyfs==3.7.0 # via @@ -1055,7 +1055,7 @@ openedx-filters==1.11.0 # -r requirements/edx/base.txt # lti-consumer-xblock # ora2 -openedx-forum==0.1.3 +openedx-forum==0.1.5 # via -r requirements/edx/base.txt openedx-learning==0.18.1 # via @@ -1069,7 +1069,7 @@ optimizely-sdk==4.1.1 # -r requirements/edx/base.txt ora2==6.14.1 # via -r requirements/edx/base.txt -packaging==24.1 +packaging==24.2 # via # -r requirements/edx/base.txt # drf-yasg @@ -1081,7 +1081,7 @@ packaging==24.1 # tox pact-python==2.2.2 # via -r requirements/edx/testing.in -pansi==2020.7.3 +pansi==2024.11.0 # via # -r requirements/edx/base.txt # py2neo @@ -1119,6 +1119,7 @@ pillow==11.0.0 # edx-enterprise # edx-organizations # edxval + # pansi platformdirs==4.3.6 # via # -r requirements/edx/base.txt @@ -1141,16 +1142,17 @@ prompt-toolkit==3.0.48 # via # -r requirements/edx/base.txt # click-repl -propcache==0.2.0 +propcache==0.2.1 # via # -r requirements/edx/base.txt + # aiohttp # yarl proto-plus==1.25.0 # via # -r requirements/edx/base.txt # google-api-core # google-cloud-firestore -protobuf==5.28.3 +protobuf==5.29.1 # via # -r requirements/edx/base.txt # google-api-core @@ -1196,12 +1198,12 @@ pycryptodomex==3.21.0 # edx-proctoring # lti-consumer-xblock # pyjwkest -pydantic==2.9.2 +pydantic==2.10.3 # via # -r requirements/edx/base.txt # camel-converter # fastapi -pydantic-core==2.23.4 +pydantic-core==2.27.1 # via # -r requirements/edx/base.txt # pydantic @@ -1216,7 +1218,7 @@ pyjwkest==1.4.2 # -r requirements/edx/base.txt # edx-token-utils # lti-consumer-xblock -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.1 # via # -r requirements/edx/base.txt # drf-jwt @@ -1270,7 +1272,7 @@ pynacl==1.5.0 # paramiko pynliner==0.8.0 # via -r requirements/edx/base.txt -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via # -r requirements/edx/base.txt # optimizely-sdk @@ -1293,7 +1295,7 @@ pysrt==1.1.2 # via # -r requirements/edx/base.txt # edxval -pytest==8.3.3 +pytest==8.3.4 # via # -r requirements/edx/testing.in # pylint-pytest @@ -1306,7 +1308,7 @@ pytest==8.3.3 # pytest-xdist pytest-attrib==0.1.3 # via -r requirements/edx/testing.in -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via -r requirements/edx/testing.in pytest-django==4.9.0 # via -r requirements/edx/testing.in @@ -1397,7 +1399,7 @@ referencing==0.35.1 # -r requirements/edx/base.txt # jsonschema # jsonschema-specifications -regex==2024.9.11 +regex==2024.11.6 # via # -r requirements/edx/base.txt # nltk @@ -1434,7 +1436,7 @@ requests-oauthlib==2.0.0 # via # -r requirements/edx/base.txt # social-auth-core -rpds-py==0.20.0 +rpds-py==0.22.3 # via # -r requirements/edx/base.txt # jsonschema @@ -1449,7 +1451,7 @@ rules==3.5 # edx-enterprise # edx-proctoring # openedx-learning -s3transfer==0.10.3 +s3transfer==0.10.4 # via # -r requirements/edx/base.txt # boto3 @@ -1477,11 +1479,10 @@ simplejson==3.19.3 # xblock-utils singledispatch==4.1.0 # via -r requirements/edx/testing.in -six==1.16.0 +six==1.17.0 # via # -r requirements/edx/base.txt # analytics-python - # bleach # codejail-includes # crowdsourcehinter-xblock # edx-ace @@ -1500,7 +1501,6 @@ six==1.16.0 # libsass # optimizely-sdk # pact-python - # pansi # paver # py2neo # pyjwkest @@ -1512,7 +1512,7 @@ slumber==0.7.1 # edx-enterprise sniffio==1.3.1 # via anyio -snowflake-connector-python==3.12.3 +snowflake-connector-python==3.12.4 # via # -r requirements/edx/base.txt # edx-enterprise @@ -1538,15 +1538,15 @@ soupsieve==2.6 # via # -r requirements/edx/base.txt # beautifulsoup4 -sqlparse==0.5.1 +sqlparse==0.5.2 # via # -r requirements/edx/base.txt # django staff-graded-xblock==2.3.0 # via -r requirements/edx/base.txt -starlette==0.41.2 +starlette==0.41.3 # via fastapi -stevedore==5.3.0 +stevedore==5.4.0 # via # -r requirements/edx/base.txt # code-annotations @@ -1571,7 +1571,7 @@ text-unidecode==1.3 # via # -r requirements/edx/base.txt # python-slugify -tinycss2==1.2.1 +tinycss2==1.4.0 # via # -r requirements/edx/base.txt # bleach @@ -1582,7 +1582,7 @@ tomlkit==0.13.2 # snowflake-connector-python tox==4.23.2 # via -r requirements/edx/testing.in -tqdm==4.66.6 +tqdm==4.67.1 # via # -r requirements/edx/base.txt # nltk @@ -1590,6 +1590,7 @@ tqdm==4.66.6 typing-extensions==4.12.2 # via # -r requirements/edx/base.txt + # anyio # django-countries # edx-opaque-keys # faker @@ -1628,7 +1629,7 @@ urllib3==2.2.3 # requests user-util==1.1.0 # via -r requirements/edx/base.txt -uvicorn==0.32.0 +uvicorn==0.32.1 # via pact-python vine==5.1.0 # via @@ -1636,7 +1637,7 @@ vine==5.1.0 # amqp # celery # kombu -virtualenv==20.27.1 +virtualenv==20.28.0 # via tox voluptuous==0.15.2 # via @@ -1646,7 +1647,7 @@ walrus==0.9.4 # via # -r requirements/edx/base.txt # edx-event-bus-redis -watchdog==5.0.3 +watchdog==6.0.0 # via -r requirements/edx/base.txt wcwidth==0.2.13 # via @@ -1670,7 +1671,7 @@ webob==1.8.9 # via # -r requirements/edx/base.txt # xblock -wrapt==1.16.0 +wrapt==1.17.0 # via # -r requirements/edx/base.txt # astroid @@ -1709,12 +1710,12 @@ xmlsec==1.3.14 # python3-saml xss-utils==0.6.0 # via -r requirements/edx/base.txt -yarl==1.17.0 +yarl==1.18.3 # via # -r requirements/edx/base.txt # aiohttp # pact-python -zipp==3.20.2 +zipp==3.21.0 # via # -r requirements/edx/base.txt # importlib-metadata diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 110663ff6ab3..c5a5c5b6fea5 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -10,7 +10,7 @@ click==8.1.6 # via # -c requirements/constraints.txt # pip-tools -packaging==24.1 +packaging==24.2 # via build pip-tools==7.4.1 # via -r requirements/pip-tools.in @@ -18,7 +18,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -wheel==0.44.0 +wheel==0.45.1 # via pip-tools # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/pip.txt b/requirements/pip.txt index 797974efa45a..0bdb9d7ac387 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -4,7 +4,7 @@ # # make upgrade # -wheel==0.44.0 +wheel==0.45.1 # via -r requirements/pip.in # The following packages are considered to be unsafe in a requirements file: @@ -12,5 +12,5 @@ pip==24.2 # via # -c requirements/common_constraints.txt # -r requirements/pip.in -setuptools==75.2.0 +setuptools==75.6.0 # via -r requirements/pip.in diff --git a/scripts/eslint.py b/scripts/eslint.py new file mode 100644 index 000000000000..3723ada6e219 --- /dev/null +++ b/scripts/eslint.py @@ -0,0 +1,73 @@ +""" # pylint: disable=django-not-configured +Check code quality using eslint. +""" + +import re +import subprocess +import shlex +import sys + + +class BuildFailure(Exception): + pass + + +def fail_quality(message): + """ + Fail the specified quality check. + """ + + raise BuildFailure(message) + + +def run_eslint(): + """ + Runs eslint on static asset directories. + If limit option is passed, fails build if more violations than the limit are found. + """ + violations_limit = 734 + + command = [ + "node", + "--max_old_space_size=4096", + "node_modules/.bin/eslint", + "--ext", ".js", + "--ext", ".jsx", + "--format=compact", + "lms", + "cms", + "common", + "openedx", + "xmodule", + ] + print("Running command:", shlex.join(command)) + result = subprocess.run( + command, + text=True, + check=False, + capture_output=True + ) + + print(result.stdout) + last_line = result.stdout.strip().splitlines()[-1] if result.stdout.strip().splitlines() else "" + regex = r'^\d+' + try: + num_violations = int(re.search(regex, last_line).group(0)) if last_line else 0 + # Fail if number of violations is greater than the limit + if num_violations > violations_limit: + fail_quality( + "FAILURE: Too many eslint violations ({count}).\nThe limit is {violations_limit}.".format(count=num_violations, violations_limit=violations_limit)) + else: + print(f"successfully run eslint with '{num_violations}' violations") + + # An AttributeError will occur if the regex finds no matches. + except (AttributeError, ValueError): + fail_quality(f"FAILURE: Number of eslint violations could not be found in '{last_line}'") + + +if __name__ == "__main__": + try: + run_eslint() + except BuildFailure as e: + print(e) + sys.exit(1) diff --git a/scripts/generic-ci-tests.sh b/scripts/generic-ci-tests.sh deleted file mode 100755 index 54b9cbb9d500..000000000000 --- a/scripts/generic-ci-tests.sh +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env bash -set -e - -############################################################################### -# -# generic-ci-tests.sh -# -# Execute some tests for edx-platform. -# (Most other tests are run by invoking `pytest`, `pylint`, etc. directly) -# -# This script can be called from CI jobs that define -# these environment variables: -# -# `TEST_SUITE` defines which kind of test to run. -# Possible values are: -# -# - "quality": Run the quality (pycodestyle/pylint) checks -# - "js-unit": Run the JavaScript tests -# - "pavelib-js-unit": Run the JavaScript tests and the Python unit -# tests from the pavelib/lib directory -# -############################################################################### - -# Clean up previous builds -git clean -qxfd - -function emptyxunit { - - cat > "reports/$1.xml" < - - - -END - -} - -# if specified tox environment is supported, prepend paver commands -# with tox env invocation -if [ -z ${TOX_ENV+x} ] || [[ ${TOX_ENV} == 'null' ]]; then - echo "TOX_ENV: ${TOX_ENV}" - TOX="" -elif tox -l |grep -q "${TOX_ENV}"; then - if [[ "${TOX_ENV}" == 'quality' ]]; then - TOX="" - else - TOX="tox -r -e ${TOX_ENV} --" - fi -else - echo "${TOX_ENV} is not currently supported. Please review the" - echo "tox.ini file to see which environments are supported" - exit 1 -fi - -PAVER_ARGS="-v" -export SUBSET_JOB=$JOB_NAME - -function run_paver_quality { - QUALITY_TASK=$1 - shift - mkdir -p test_root/log/ - LOG_PREFIX="test_root/log/$QUALITY_TASK" - $TOX paver "$QUALITY_TASK" "$@" 2> "$LOG_PREFIX.err.log" > "$LOG_PREFIX.out.log" || { - echo "STDOUT (last 100 lines of $LOG_PREFIX.out.log):"; - tail -n 100 "$LOG_PREFIX.out.log" - echo "STDERR (last 100 lines of $LOG_PREFIX.err.log):"; - tail -n 100 "$LOG_PREFIX.err.log" - return 1; - } - return 0; -} - -case "$TEST_SUITE" in - - "quality") - EXIT=0 - - mkdir -p reports - - echo "Finding pycodestyle violations and storing report..." - run_paver_quality run_pep8 || { EXIT=1; } - echo "Finding ESLint violations and storing report..." - run_paver_quality run_eslint -l "$ESLINT_THRESHOLD" || { EXIT=1; } - echo "Finding Stylelint violations and storing report..." - run_paver_quality run_stylelint || { EXIT=1; } - echo "Running xss linter report." - run_paver_quality run_xsslint -t "$XSSLINT_THRESHOLDS" || { EXIT=1; } - echo "Running PII checker on all Django models..." - run_paver_quality run_pii_check || { EXIT=1; } - echo "Running reserved keyword checker on all Django models..." - run_paver_quality check_keywords || { EXIT=1; } - - # Need to create an empty test result so the post-build - # action doesn't fail the build. - emptyxunit "stub" - exit "$EXIT" - ;; - - "js-unit") - $TOX paver test_js --coverage - $TOX paver diff_coverage - ;; - - "pavelib-js-unit") - EXIT=0 - $TOX paver test_js --coverage --skip-clean || { EXIT=1; } - paver test_lib --skip-clean $PAVER_ARGS || { EXIT=1; } - - # This is to ensure that the build status of the shard is properly set. - # Because we are running two paver commands in a row, we need to capture - # their return codes in order to exit with a non-zero code if either of - # them fail. We put the || clause there because otherwise, when a paver - # command fails, this entire script will exit, and not run the second - # paver command in this case statement. So instead of exiting, the value - # of a variable named EXIT will be set to 1 if either of the paver - # commands fail. We then use this variable's value as our exit code. - # Note that by default the value of this variable EXIT is not set, so if - # neither command fails then the exit command resolves to simply exit - # which is considered successful. - exit "$EXIT" - ;; -esac diff --git a/scripts/structures_pruning/requirements/base.txt b/scripts/structures_pruning/requirements/base.txt index a3fcacad2f7e..b15cb81f29ee 100644 --- a/scripts/structures_pruning/requirements/base.txt +++ b/scripts/structures_pruning/requirements/base.txt @@ -22,7 +22,7 @@ pymongo==4.4.0 # -c scripts/structures_pruning/requirements/../../../requirements/constraints.txt # -r scripts/structures_pruning/requirements/base.in # edx-opaque-keys -stevedore==5.3.0 +stevedore==5.4.0 # via edx-opaque-keys typing-extensions==4.12.2 # via edx-opaque-keys diff --git a/scripts/structures_pruning/requirements/testing.txt b/scripts/structures_pruning/requirements/testing.txt index 94c6ac6982f3..e121a05143fa 100644 --- a/scripts/structures_pruning/requirements/testing.txt +++ b/scripts/structures_pruning/requirements/testing.txt @@ -20,7 +20,7 @@ edx-opaque-keys==2.11.0 # via -r scripts/structures_pruning/requirements/base.txt iniconfig==2.0.0 # via pytest -packaging==24.1 +packaging==24.2 # via pytest pbr==6.1.0 # via @@ -32,9 +32,9 @@ pymongo==4.4.0 # via # -r scripts/structures_pruning/requirements/base.txt # edx-opaque-keys -pytest==8.3.3 +pytest==8.3.4 # via -r scripts/structures_pruning/requirements/testing.in -stevedore==5.3.0 +stevedore==5.4.0 # via # -r scripts/structures_pruning/requirements/base.txt # edx-opaque-keys diff --git a/scripts/user_retirement/requirements/base.txt b/scripts/user_retirement/requirements/base.txt index 704baaff2c79..622ffd2bc135 100644 --- a/scripts/user_retirement/requirements/base.txt +++ b/scripts/user_retirement/requirements/base.txt @@ -10,9 +10,9 @@ attrs==24.2.0 # via zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.in -boto3==1.35.50 +boto3==1.35.76 # via -r scripts/user_retirement/requirements/base.in -botocore==1.35.50 +botocore==1.35.76 # via # boto3 # s3transfer @@ -33,9 +33,9 @@ click==8.1.6 # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt # -r scripts/user_retirement/requirements/base.in # edx-django-utils -cryptography==43.0.3 +cryptography==44.0.0 # via pyjwt -django==4.2.16 +django==4.2.17 # via # -c scripts/user_retirement/requirements/../../../requirements/common_constraints.txt # -c scripts/user_retirement/requirements/../../../requirements/constraints.txt @@ -44,24 +44,24 @@ django==4.2.16 # edx-django-utils django-crum==0.7.9 # via edx-django-utils -django-waffle==4.1.0 +django-waffle==4.2.0 # via edx-django-utils -edx-django-utils==7.0.0 +edx-django-utils==7.1.0 # via edx-rest-api-client edx-rest-api-client==6.0.0 # via -r scripts/user_retirement/requirements/base.in -google-api-core==2.22.0 +google-api-core==2.23.0 # via google-api-python-client -google-api-python-client==2.149.0 +google-api-python-client==2.154.0 # via -r scripts/user_retirement/requirements/base.in -google-auth==2.35.0 +google-auth==2.36.0 # via # google-api-core # google-api-python-client # google-auth-httplib2 google-auth-httplib2==0.2.0 # via google-api-python-client -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via google-api-core httplib2==0.22.0 # via @@ -81,7 +81,7 @@ lxml==5.3.0 # via zeep more-itertools==10.5.0 # via simple-salesforce -newrelic==10.2.0 +newrelic==10.3.1 # via edx-django-utils pbr==6.1.0 # via stevedore @@ -89,7 +89,7 @@ platformdirs==4.3.6 # via zeep proto-plus==1.25.0 # via google-api-core -protobuf==5.28.3 +protobuf==5.29.1 # via # google-api-core # googleapis-common-protos @@ -104,7 +104,7 @@ pyasn1-modules==0.4.1 # via google-auth pycparser==2.22 # via cffi -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.1 # via # edx-rest-api-client # simple-salesforce @@ -136,19 +136,19 @@ requests-toolbelt==1.0.0 # via zeep rsa==4.9 # via google-auth -s3transfer==0.10.3 +s3transfer==0.10.4 # via boto3 simple-salesforce==1.12.6 # via -r scripts/user_retirement/requirements/base.in simplejson==3.19.3 # via -r scripts/user_retirement/requirements/base.in -six==1.16.0 +six==1.17.0 # via # jenkinsapi # python-dateutil -sqlparse==0.5.1 +sqlparse==0.5.2 # via django -stevedore==5.3.0 +stevedore==5.4.0 # via edx-django-utils typing-extensions==4.12.2 # via simple-salesforce diff --git a/scripts/user_retirement/requirements/testing.txt b/scripts/user_retirement/requirements/testing.txt index 4cb3de607db7..efaa4369170f 100644 --- a/scripts/user_retirement/requirements/testing.txt +++ b/scripts/user_retirement/requirements/testing.txt @@ -14,11 +14,11 @@ attrs==24.2.0 # zeep backoff==2.2.1 # via -r scripts/user_retirement/requirements/base.txt -boto3==1.35.50 +boto3==1.35.76 # via # -r scripts/user_retirement/requirements/base.txt # moto -botocore==1.35.50 +botocore==1.35.76 # via # -r scripts/user_retirement/requirements/base.txt # boto3 @@ -45,14 +45,14 @@ click==8.1.6 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -cryptography==43.0.3 +cryptography==44.0.0 # via # -r scripts/user_retirement/requirements/base.txt # moto # pyjwt ddt==1.7.2 # via -r scripts/user_retirement/requirements/testing.in -django==4.2.16 +django==4.2.17 # via # -r scripts/user_retirement/requirements/base.txt # django-crum @@ -62,23 +62,23 @@ django-crum==0.7.9 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -django-waffle==4.1.0 +django-waffle==4.2.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -edx-django-utils==7.0.0 +edx-django-utils==7.1.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-rest-api-client edx-rest-api-client==6.0.0 # via -r scripts/user_retirement/requirements/base.txt -google-api-core==2.22.0 +google-api-core==2.23.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-python-client -google-api-python-client==2.149.0 +google-api-python-client==2.154.0 # via -r scripts/user_retirement/requirements/base.txt -google-auth==2.35.0 +google-auth==2.36.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -88,7 +88,7 @@ google-auth-httplib2==0.2.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-python-client -googleapis-common-protos==1.65.0 +googleapis-common-protos==1.66.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -132,11 +132,11 @@ more-itertools==10.5.0 # simple-salesforce moto==4.2.14 # via -r scripts/user_retirement/requirements/testing.in -newrelic==10.2.0 +newrelic==10.3.1 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils -packaging==24.1 +packaging==24.2 # via pytest pbr==6.1.0 # via @@ -152,7 +152,7 @@ proto-plus==1.25.0 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core -protobuf==5.28.3 +protobuf==5.29.1 # via # -r scripts/user_retirement/requirements/base.txt # google-api-core @@ -175,7 +175,7 @@ pycparser==2.22 # via # -r scripts/user_retirement/requirements/base.txt # cffi -pyjwt[crypto]==2.9.0 +pyjwt[crypto]==2.10.1 # via # -r scripts/user_retirement/requirements/base.txt # edx-rest-api-client @@ -188,7 +188,7 @@ pyparsing==3.2.0 # via # -r scripts/user_retirement/requirements/base.txt # httplib2 -pytest==8.3.3 +pytest==8.3.4 # via -r scripts/user_retirement/requirements/testing.in python-dateutil==2.9.0.post0 # via @@ -235,7 +235,7 @@ rsa==4.9 # via # -r scripts/user_retirement/requirements/base.txt # google-auth -s3transfer==0.10.3 +s3transfer==0.10.4 # via # -r scripts/user_retirement/requirements/base.txt # boto3 @@ -243,16 +243,16 @@ simple-salesforce==1.12.6 # via -r scripts/user_retirement/requirements/base.txt simplejson==3.19.3 # via -r scripts/user_retirement/requirements/base.txt -six==1.16.0 +six==1.17.0 # via # -r scripts/user_retirement/requirements/base.txt # jenkinsapi # python-dateutil -sqlparse==0.5.1 +sqlparse==0.5.2 # via # -r scripts/user_retirement/requirements/base.txt # django -stevedore==5.3.0 +stevedore==5.4.0 # via # -r scripts/user_retirement/requirements/base.txt # edx-django-utils @@ -272,7 +272,7 @@ urllib3==1.26.20 # botocore # requests # responses -werkzeug==3.0.6 +werkzeug==3.1.3 # via moto xmltodict==0.14.2 # via moto diff --git a/scripts/xsslint/xss_linter.py b/scripts/xsslint/xss_linter.py index a35038c3de6d..b32c54aa5cd7 100755 --- a/scripts/xsslint/xss_linter.py +++ b/scripts/xsslint/xss_linter.py @@ -4,6 +4,316 @@ """ +import argparse +import importlib +import json +import os +import re +import sys + +from functools import reduce +from io import StringIO +from xsslint.reporting import SummaryResults +from xsslint.rules import RuleSet +from xsslint.utils import is_skip_dir + + +class BuildFailure(Exception): + pass + + +def fail_quality(message): + """ + Fail the specified quality check. + """ + + raise BuildFailure(message) + + +def _load_config_module(module_path): + cwd = os.getcwd() + if cwd not in sys.path: + # Enable config module to be imported relative to wherever the script was run from. + sys.path.append(cwd) + return importlib.import_module(module_path) + + +def _build_ruleset(template_linters): + """ + Combines the RuleSets from the provided template_linters into a single, aggregate RuleSet. + + Arguments: + template_linters: A list of linting objects. + + Returns: + The combined RuleSet. + """ + return reduce( + lambda combined, current: combined + current.ruleset, + template_linters, + RuleSet() + ) + + +def _process_file(full_path, template_linters, options, summary_results, out): + """ + For each linter, lints the provided file. This means finding and printing + violations. + + Arguments: + full_path: The full path of the file to lint. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + num_violations = 0 + directory = os.path.dirname(full_path) + file_name = os.path.basename(full_path) + try: + for template_linter in template_linters: + results = template_linter.process_file(directory, file_name) + results.print_results(options, summary_results, out) + except BaseException as e: + raise Exception(f"Failed to process path: {full_path}") from e + + +def _process_os_dir(directory, files, template_linters, options, summary_results, out): + """ + Calls out to lint each file in the passed list of files. + + Arguments: + directory: Directory being linted. + files: All files in the directory to be linted. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + for current_file in sorted(files, key=lambda s: s.lower()): + full_path = os.path.join(directory, current_file) + _process_file(full_path, template_linters, options, summary_results, out) + + +def _process_os_dirs(starting_dir, template_linters, options, summary_results, out): + """ + For each linter, lints all the directories in the starting directory. + + Arguments: + starting_dir: The initial directory to begin the walk. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + skip_dirs = options.get('skip_dirs', ()) + for root, dirs, files in os.walk(starting_dir): + if is_skip_dir(skip_dirs, root): + del dirs + continue + dirs.sort(key=lambda s: s.lower()) + _process_os_dir(root, files, template_linters, options, summary_results, out) + + +def _get_xsslint_counts(result_contents): + """ + This returns a dict of violations from the xsslint report. + + Arguments: + filename: The name of the xsslint report. + + Returns: + A dict containing the following: + rules: A dict containing the count for each rule as follows: + violation-rule-id: N, where N is the number of violations + total: M, where M is the number of total violations + + """ + + rule_count_regex = re.compile(r"^(?P[a-z-]+):\s+(?P\d+) violations", re.MULTILINE) + total_count_regex = re.compile(r"^(?P\d+) violations total", re.MULTILINE) + violations = {'rules': {}} + for violation_match in rule_count_regex.finditer(result_contents): + try: + violations['rules'][violation_match.group('rule_id')] = int(violation_match.group('count')) + except ValueError: + violations['rules'][violation_match.group('rule_id')] = None + try: + violations['total'] = int(total_count_regex.search(result_contents).group('count')) + # An AttributeError will occur if the regex finds no matches. + # A ValueError will occur if the returned regex cannot be cast as a float. + except (AttributeError, ValueError): + violations['total'] = None + return violations + + +def _check_violations(options, results): + xsslint_script = "xss_linter.py" + try: + thresholds_option = options['thresholds'] + # Read the JSON file + with open(thresholds_option, 'r') as file: + violation_thresholds = json.load(file) + + except ValueError: + violation_thresholds = None + if isinstance(violation_thresholds, dict) is False or \ + any(key not in ("total", "rules") for key in violation_thresholds.keys()): + print('xsslint') + fail_quality("""FAILURE: Thresholds option "{thresholds_option}" was not supplied using proper format.\n""" + """Here is a properly formatted example, '{{"total":100,"rules":{{"javascript-escape":0}}}}' """ + """with property names in double-quotes.""".format(thresholds_option=thresholds_option)) + + try: + metrics_str = "Number of {xsslint_script} violations: {num_violations}\n".format( + xsslint_script=xsslint_script, num_violations=int(results['total']) + ) + if 'rules' in results and any(results['rules']): + metrics_str += "\n" + rule_keys = sorted(results['rules'].keys()) + for rule in rule_keys: + metrics_str += "{rule} violations: {count}\n".format( + rule=rule, + count=int(results['rules'][rule]) + ) + except TypeError: + print('xsslint') + fail_quality("FAILURE: Number of {xsslint_script} violations could not be found".format( + xsslint_script=xsslint_script + )) + + error_message = "" + # Test total violations against threshold. + if 'total' in list(violation_thresholds.keys()): + if violation_thresholds['total'] < results['total']: + error_message = "Too many violations total ({count}).\nThe limit is {violations_limit}.".format( + count=results['total'], violations_limit=violation_thresholds['total'] + ) + + # Test rule violations against thresholds. + if 'rules' in violation_thresholds: + threshold_keys = sorted(violation_thresholds['rules'].keys()) + for threshold_key in threshold_keys: + if threshold_key not in results['rules']: + error_message += ( + "\nNumber of {xsslint_script} violations for {rule} could not be found" + ).format( + xsslint_script=xsslint_script, rule=threshold_key + ) + elif violation_thresholds['rules'][threshold_key] < results['rules'][threshold_key]: + error_message += \ + "\nToo many {rule} violations ({count}).\nThe {rule} limit is {violations_limit}.".format( + rule=threshold_key, count=results['rules'][threshold_key], + violations_limit=violation_thresholds['rules'][threshold_key], + ) + + if error_message: + print('xsslint') + fail_quality("FAILURE: XSSLinter Failed.\n{error_message}\n" + "run the following command to hone in on the problem:\n" + "./scripts/xss-commit-linter.sh -h".format(error_message=error_message)) + else: + print("successfully run xsslint") + + +def _lint(file_or_dir, template_linters, options, summary_results, out): + """ + For each linter, lints the provided file or directory. + + Arguments: + file_or_dir: The file or initial directory to lint. + template_linters: A list of linting objects. + options: A list of the options. + summary_results: A SummaryResults with a summary of the violations. + out: output file + + """ + + if file_or_dir is not None and os.path.isfile(file_or_dir): + _process_file(file_or_dir, template_linters, options, summary_results, out) + else: + directory = "." + if file_or_dir is not None: + if os.path.exists(file_or_dir): + directory = file_or_dir + else: + raise ValueError(f"Path [{file_or_dir}] is not a valid file or directory.") + _process_os_dirs(directory, template_linters, options, summary_results, out) + + summary_results.print_results(options, out) + result_output = _get_xsslint_counts(out.getvalue()) + _check_violations(options, result_output) + + +def main(): + """ + Used to execute the linter. Use --help option for help. + + Prints all violations. + """ + epilog = "For more help using the xss linter, including details on how to\n" + epilog += "understand and fix any violations, read the docs here:\n" + epilog += "\n" + # pylint: disable=line-too-long + epilog += " https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/preventing_xss.html#xss-linter\n" + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description='Checks that templates are safe.', + epilog=epilog, + ) + parser.add_argument( + '--list-files', dest='list_files', action='store_true', + help='Only display the filenames that contain violations.' + ) + parser.add_argument( + '--rule-totals', dest='rule_totals', action='store_true', + help='Display the totals for each rule.' + ) + parser.add_argument( + '--summary-format', dest='summary_format', + choices=['eslint', 'json'], default='eslint', + help='Choose the display format for the summary.' + ) + parser.add_argument( + '--verbose', dest='verbose', action='store_true', + help='Print multiple lines where possible for additional context of violations.' + ) + parser.add_argument( + '--config', dest='config', action='store', default='xsslint.default_config', + help='Specifies the config module to use. The config module should be in Python package syntax.' + ) + parser.add_argument( + '--thresholds', dest='thresholds', action='store', + help='Specifies the config module to use. The config module should be in Python package syntax.' + ) + parser.add_argument('path', nargs="?", default=None, help='A file to lint or directory to recursively lint.') + + args = parser.parse_args() + config = _load_config_module(args.config) + options = { + 'list_files': args.list_files, + 'rule_totals': args.rule_totals, + 'summary_format': args.summary_format, + 'verbose': args.verbose, + 'skip_dirs': getattr(config, 'SKIP_DIRS', ()), + 'thresholds': args.thresholds + } + template_linters = getattr(config, 'LINTERS', ()) + if not template_linters: + raise ValueError(f"LINTERS is empty or undefined in the config module ({args.config}).") + + ruleset = _build_ruleset(template_linters) + summary_results = SummaryResults(ruleset) + _lint(args.path, template_linters, options, summary_results, out=StringIO()) + + if __name__ == "__main__": - from xsslint.main import main - main() + try: + main() + except BuildFailure as e: + print(e) + sys.exit(1) diff --git a/scripts/xsslint/xsslint/main.py b/scripts/xsslint/xsslint/main.py deleted file mode 100644 index f8f8672b74b3..000000000000 --- a/scripts/xsslint/xsslint/main.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -The main function for the XSS linter. -""" - - -import argparse -import importlib -import os -import sys -from functools import reduce - -from xsslint.reporting import SummaryResults -from xsslint.rules import RuleSet -from xsslint.utils import is_skip_dir - - -def _load_config_module(module_path): - cwd = os.getcwd() - if cwd not in sys.path: - # Enable config module to be imported relative to wherever the script was run from. - sys.path.append(cwd) - return importlib.import_module(module_path) - - -def _build_ruleset(template_linters): - """ - Combines the RuleSets from the provided template_linters into a single, aggregate RuleSet. - - Arguments: - template_linters: A list of linting objects. - - Returns: - The combined RuleSet. - """ - return reduce( - lambda combined, current: combined + current.ruleset, - template_linters, - RuleSet() - ) - - -def _process_file(full_path, template_linters, options, summary_results, out): - """ - For each linter, lints the provided file. This means finding and printing - violations. - - Arguments: - full_path: The full path of the file to lint. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - num_violations = 0 - directory = os.path.dirname(full_path) - file_name = os.path.basename(full_path) - try: - for template_linter in template_linters: - results = template_linter.process_file(directory, file_name) - results.print_results(options, summary_results, out) - except BaseException as e: - raise Exception(f"Failed to process path: {full_path}") from e - - -def _process_os_dir(directory, files, template_linters, options, summary_results, out): - """ - Calls out to lint each file in the passed list of files. - - Arguments: - directory: Directory being linted. - files: All files in the directory to be linted. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - for current_file in sorted(files, key=lambda s: s.lower()): - full_path = os.path.join(directory, current_file) - _process_file(full_path, template_linters, options, summary_results, out) - - -def _process_os_dirs(starting_dir, template_linters, options, summary_results, out): - """ - For each linter, lints all the directories in the starting directory. - - Arguments: - starting_dir: The initial directory to begin the walk. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - skip_dirs = options.get('skip_dirs', ()) - for root, dirs, files in os.walk(starting_dir): - if is_skip_dir(skip_dirs, root): - del dirs - continue - dirs.sort(key=lambda s: s.lower()) - _process_os_dir(root, files, template_linters, options, summary_results, out) - - -def _lint(file_or_dir, template_linters, options, summary_results, out): - """ - For each linter, lints the provided file or directory. - - Arguments: - file_or_dir: The file or initial directory to lint. - template_linters: A list of linting objects. - options: A list of the options. - summary_results: A SummaryResults with a summary of the violations. - out: output file - - """ - - if file_or_dir is not None and os.path.isfile(file_or_dir): - _process_file(file_or_dir, template_linters, options, summary_results, out) - else: - directory = "." - if file_or_dir is not None: - if os.path.exists(file_or_dir): - directory = file_or_dir - else: - raise ValueError(f"Path [{file_or_dir}] is not a valid file or directory.") - _process_os_dirs(directory, template_linters, options, summary_results, out) - - summary_results.print_results(options, out) - - -def main(): - """ - Used to execute the linter. Use --help option for help. - - Prints all violations. - """ - epilog = "For more help using the xss linter, including details on how to\n" - epilog += "understand and fix any violations, read the docs here:\n" - epilog += "\n" - # pylint: disable=line-too-long - epilog += " https://edx.readthedocs.org/projects/edx-developer-guide/en/latest/conventions/preventing_xss.html#xss-linter\n" - - parser = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description='Checks that templates are safe.', - epilog=epilog, - ) - parser.add_argument( - '--list-files', dest='list_files', action='store_true', - help='Only display the filenames that contain violations.' - ) - parser.add_argument( - '--rule-totals', dest='rule_totals', action='store_true', - help='Display the totals for each rule.' - ) - parser.add_argument( - '--summary-format', dest='summary_format', - choices=['eslint', 'json'], default='eslint', - help='Choose the display format for the summary.' - ) - parser.add_argument( - '--verbose', dest='verbose', action='store_true', - help='Print multiple lines where possible for additional context of violations.' - ) - parser.add_argument( - '--config', dest='config', action='store', default='xsslint.default_config', - help='Specifies the config module to use. The config module should be in Python package syntax.' - ) - parser.add_argument('path', nargs="?", default=None, help='A file to lint or directory to recursively lint.') - - args = parser.parse_args() - config = _load_config_module(args.config) - options = { - 'list_files': args.list_files, - 'rule_totals': args.rule_totals, - 'summary_format': args.summary_format, - 'verbose': args.verbose, - 'skip_dirs': getattr(config, 'SKIP_DIRS', ()) - } - template_linters = getattr(config, 'LINTERS', ()) - if not template_linters: - raise ValueError(f"LINTERS is empty or undefined in the config module ({args.config}).") - - ruleset = _build_ruleset(template_linters) - summary_results = SummaryResults(ruleset) - _lint(args.path, template_linters, options, summary_results, out=sys.stdout) diff --git a/stylelint.config.js b/stylelint.config.js deleted file mode 100644 index bd7769911708..000000000000 --- a/stylelint.config.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - extends: '@edx/stylelint-config-edx' -}; diff --git a/tox.ini b/tox.ini index 1b4252fd1905..e5df7f0fbd2f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py{38,311} quality +envlist = py{311} quality # This is needed to prevent the lms, cms, and openedx packages inside the "Open # edX" package (defined in setup.py) from getting installed into site-packages diff --git a/xmodule/templates/html/zooming_image.yaml b/xmodule/templates/html/zooming_image.yaml deleted file mode 100644 index b91717550b27..000000000000 --- a/xmodule/templates/html/zooming_image.yaml +++ /dev/null @@ -1,57 +0,0 @@ ---- -metadata: - display_name: Zooming Image Tool -data: | -

Zooming Image Tool

-

Use the Zooming Image Tool to enable learners to see details of large, complex images.

-

With the Zooming Image Tool, the learner can move the mouse pointer over a part of the image to enlarge it and see more detail.

-

To use the Zooming Image Tool, you must first add the jquery.loupeAndLightbox.js JavaScript file to your course.

-

You must also add both the regular and magnified image files to your course.

-

The following HTML code shows the format required to use the Zooming Image tool. For the example in this template, you must replace the values in italics.

- 
-        <div class="zooming-image-place" style="position: relative;">
-          <a class="loupe" href="path to the magnified version of the image">
-            <img alt="Text for screen readers"
-              src="path to the image you want to display in the unit" />
-          </a>
-          <div class="script_placeholder"
-            data-src="path to the jquery.loupeAndLightbox.js JavaScript file in your course"/>
-        </div>
-        <script type="text/javascript">// >![CDATA[
-        JavascriptLoader.executeModuleScripts($('.zooming-image-place').eq(0), function() {
-          $('.loupe').loupeAndLightbox({
-            width: 350,
-            height: 350,
-            lightbox: false
-          });
-        });
-        // ]]></script>
-        <div id="ap_listener_added"></div>
-
- -

You can modify the example below for your own use.

-
    -
  1. Replace the value of the link's href attribute with the path to the magnified image. Do not change the value of the class attribute.
    2. -
  2. Replace the value of the image's src attribute with the path to the image that will appear in the unit.
    3. -
  3. Replace the value of the image's alt attribute with text that both describes the image and the action or destination of clicking on the image. You must include alt text to provide an accessible label.
    4. -
  4. Replace the value of the div element's data-src attribute with the path to the jquery.loupeAndLightbox.js JavaScript file in your course.
    5. -
-

The example below shows a subset of the biochemical reactions that cells carry out.

-

You can view the chemical structures of the molecules by clicking on them. The magnified view also lists the enzymes involved in each step.

-

Press spacebar to open the magnifier.

-
- - magnify - -
-
- -
diff --git a/xmodule/templates/test/zooming_image.yaml b/xmodule/templates/test/zooming_image.yaml deleted file mode 100644 index 3ac9d63bcbbc..000000000000 --- a/xmodule/templates/test/zooming_image.yaml +++ /dev/null @@ -1,25 +0,0 @@ ---- -metadata: - display_name: Zooming Image -data: | -

ZOOMING DIAGRAMS

-

Some edX classes use extremely large, extremely detailed graphics. To make it easier to understand we can offer two versions of those graphics, with the zoomed section showing when you click on the main view.

-

The example below is from 7.00x: Introduction to Biology and shows a subset of the biochemical reactions that cells carry out.

-

You can view the chemical structures of the molecules by clicking on them. The magnified view also lists the enzymes involved in each step.

-

Press spacebar to open the magifier.

-
- - magnify - -
-
- -
diff --git a/xmodule/tests/test_resource_templates.py b/xmodule/tests/test_resource_templates.py index 742a7e9da199..470be9551489 100644 --- a/xmodule/tests/test_resource_templates.py +++ b/xmodule/tests/test_resource_templates.py @@ -18,7 +18,6 @@ class ResourceTemplatesTests(unittest.TestCase): def test_templates(self): expected = { 'latex_html.yaml', - 'zooming_image.yaml', 'announcement.yaml', 'anon_user_id.yaml'} got = {t['template_id'] for t in TestClass.templates()} @@ -38,7 +37,6 @@ def test_get_custom_template(self): def test_custom_templates(self): expected = { 'latex_html.yaml', - 'zooming_image.yaml', 'announcement.yaml', 'anon_user_id.yaml'} got = {t['template_id'] for t in TestClassResourceTemplate.templates()}