diff --git a/.babelrc b/.babelrc new file mode 100644 index 000000000000..91cf362dc6bf --- /dev/null +++ b/.babelrc @@ -0,0 +1,16 @@ +{ + "presets": [ + [ + "env", + { + "targets": { + "browsers": [ + "last 2 versions", + "IE >= 11" + ] + }, + "modules": false + } + ] + ] +} diff --git a/.coveragerc b/.coveragerc index 260d9c8e50d9..a974b527f2e9 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,7 +8,7 @@ source = common/lib/capa common/lib/xmodule lms - openedx/core/djangoapps + openedx pavelib omit = diff --git a/.eslintignore b/.eslintignore index 7bec88e2cb3e..8277bea27284 100644 --- a/.eslintignore +++ b/.eslintignore @@ -58,3 +58,5 @@ common/lib/xmodule/xmodule/js/src/vertical/edit.js # This file is responsible for almost half of the repo's total issues. common/lib/xmodule/xmodule/js/src/capa/schematic.js + +!**/.eslintrc.js diff --git a/.eslintrc.json b/.eslintrc.json index 36d4a7a6261a..c76fcbe5bf58 100644 --- a/.eslintrc.json +++ b/.eslintrc.json @@ -1,5 +1,5 @@ { - "extends": "eslint-config-edx", + "extends": "eslint-config-edx-es5", "globals": { // Try to avoid adding any new globals. // Old compatibility things and hacks "edx": true, diff --git a/.gitignore b/.gitignore index a1d2c4c1f53f..f50ad217f395 100644 --- a/.gitignore +++ b/.gitignore @@ -75,6 +75,10 @@ jscover.log.* .tddium* common/test/data/test_unicode/static/ test_root/courses/ +test_root/data/test_bare.git/ +test_root/export_course_repos/ +test_root/paver_logs/ +test_root/uploads/ django-pyfs ### Installation artifacts @@ -91,6 +95,8 @@ lms/static/css/ lms/static/certificates/css/ cms/static/css/ common/static/common/js/vendor/ +common/static/bundles +webpack-stats.json ### Styling generated from templates lms/static/sass/*.css diff --git a/.tx/config b/.tx/config index 25f0c77fc1b1..459a4ae7c366 100644 --- a/.tx/config +++ b/.tx/config @@ -55,13 +55,13 @@ source_file = conf/locale/en/LC_MESSAGES/wiki.po source_lang = en type = PO -[open-edx-releases.release-ficus] +[open-edx-releases.release-ginkgo] file_filter = conf/locale//LC_MESSAGES/django.po source_file = conf/locale/en/LC_MESSAGES/django.po source_lang = en type = PO - -[open-edx-releases.release-ficus-js] + +[open-edx-releases.release-ginkgo-js] file_filter = conf/locale//LC_MESSAGES/djangojs.po source_file = conf/locale/en/LC_MESSAGES/djangojs.po source_lang = en @@ -97,9 +97,9 @@ source_file = conf/locale/en/LC_MESSAGES/theme.po source_lang = en type = PO -[stanford-openedx.tos] -file_filter = conf/locale//LC_MESSAGES/tos.po -source_file = conf/locale/en/LC_MESSAGES/tos.po +[stanford-openedx.tos_and_honor] +file_filter = conf/locale//LC_MESSAGES/tos_and_honor.po +source_file = conf/locale/en/LC_MESSAGES/tos_and_honor.po source_lang = en type = PO @@ -109,12 +109,6 @@ source_file = conf/locale/en/LC_MESSAGES/privacy.po source_lang = en type = PO -[stanford-openedx.honor] -file_filter = conf/locale//LC_MESSAGES/honor.po -source_file = conf/locale/en/LC_MESSAGES/honor.po -source_lang = en -type = PO - [stanford-openedx.copyright] file_filter = conf/locale//LC_MESSAGES/copyright.po source_file = conf/locale/en/LC_MESSAGES/copyright.po diff --git a/AUTHORS b/AUTHORS index 3bdc31225cb7..1d6708a2c61b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -279,4 +279,7 @@ Jhony Avella Tanmay Mohapatra Brian Mesick Jeff LaJoie +Ivan Ivić +Brandon Baker +Shirley He Sahar Markovich diff --git a/Makefile b/Makefile index 98679b661a8d..4ad2e4a09b32 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,14 @@ # Do things in edx-platform +# Careful with mktemp syntax: it has to work on Mac and Ubuntu, which have differences. +PRIVATE_FILES := $(shell mktemp -u /tmp/private_files.XXXXXX) + clean: # Remove all the git-ignored stuff, but save and restore things marked - # by start-noclean/end-noclean. + # by start-noclean/end-noclean. Include Makefile in the tarball so that + # there's always at least one file even if there are no private files. sed -n -e '/start-noclean/,/end-noclean/p' < .gitignore > /tmp/private-files - tar cf /tmp/private.tar `git ls-files --exclude-from=/tmp/private-files --ignored --others` - git clean -fdX - tar xf /tmp/private.tar + -tar cf $(PRIVATE_FILES) Makefile `git ls-files --exclude-from=/tmp/private-files --ignored --others` + -git clean -fdX + tar xf $(PRIVATE_FILES) + rm $(PRIVATE_FILES) diff --git a/README.rst b/README.rst index 5df62bd254e1..ccf7f0533800 100644 --- a/README.rst +++ b/README.rst @@ -51,21 +51,9 @@ in. If you do not have an account, follow these steps. Documentation ------------- -Documentation is managed in the `edx-documentation`_ repository. Documentation -is built using `Sphinx`_: you can `view the built documentation on -ReadTheDocs`_. - -You can also check out `Confluence`_, our wiki system. Once you sign up for -an account, you'll be able to create new pages and edit existing pages, just -like in any other wiki system. You only need one account for both Confluence -and `JIRA`_, our issue tracker. - -.. _Sphinx: http://sphinx-doc.org/ -.. _view the built documentation on ReadTheDocs: http://docs.edx.org/ -.. _edx-documentation: https://github.com/edx/edx-documentation -.. _Confluence: http://openedx.atlassian.net/wiki/ -.. _JIRA: https://openedx.atlassian.net/ +Documentation details can be found in the `docs index.rst`_. +.. _docs index.rst: docs/index.rst Getting Help ------------ diff --git a/circle.yml b/circle.yml index 0bd6322b597e..189560352dad 100644 --- a/circle.yml +++ b/circle.yml @@ -13,7 +13,7 @@ dependencies: - sudo apt-get install libxmlsec1-dev - sudo apt-get install lynx-cur override: - - npm install + - npm install || npm prune && npm install - pip install setuptools - pip install --exists-action w -r requirements/edx/paver.txt diff --git a/cms/__init__.py b/cms/__init__.py index 7ca1196bb08c..294b4436cecb 100644 --- a/cms/__init__.py +++ b/cms/__init__.py @@ -4,6 +4,17 @@ """ from __future__ import absolute_import +# We monkey patch Kombu's entrypoints listing because scanning through this +# accounts for the majority of LMS/Studio startup time for tests, and we don't +# use custom Kombu serializers (which is what this is for). Still, this is +# pretty evil, and should be taken out when we update Celery to the next version +# where it looks like this method of custom serialization has been removed. +# +# FWIW, this is identical behavior to what happens in Kombu if pkg_resources +# isn't available. +import kombu.utils +kombu.utils.entrypoints = lambda namespace: iter([]) + # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import APP as CELERY_APP diff --git a/cms/celery.py b/cms/celery.py index e35bf4d7c14d..0d7505cfa3d2 100644 --- a/cms/celery.py +++ b/cms/celery.py @@ -5,9 +5,12 @@ Taken from: http://celery.readthedocs.org/en/latest/django/first-steps-with-django.html """ from __future__ import absolute_import + import os + from celery import Celery from django.conf import settings + from openedx.core.lib.celery.routers import AlternateEnvironmentRouter # set the default Django settings module for the 'celery' program. @@ -33,4 +36,6 @@ def alternate_env_tasks(self): """ return { 'openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache': 'lms', + 'openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache_v2': 'lms', + 'lms.djangoapps.grades.tasks.compute_all_grades_for_course': 'lms', } diff --git a/docs/en_us/enrollment_api/__init__.py b/cms/djangoapps/cms_user_tasks/__init__.py similarity index 100% rename from docs/en_us/enrollment_api/__init__.py rename to cms/djangoapps/cms_user_tasks/__init__.py diff --git a/cms/djangoapps/cms_user_tasks/apps.py b/cms/djangoapps/cms_user_tasks/apps.py new file mode 100644 index 000000000000..ae496cb14a13 --- /dev/null +++ b/cms/djangoapps/cms_user_tasks/apps.py @@ -0,0 +1,19 @@ +""" +CMS user tasks application configuration +Signal handlers are connected here. +""" + +from django.apps import AppConfig + + +class CmsUserTasksConfig(AppConfig): + """ + Application Configuration for cms_user_tasks. + """ + name = u'cms_user_tasks' + + def ready(self): + """ + Connect signal handlers. + """ + from . import signals # pylint: disable=unused-variable diff --git a/cms/djangoapps/cms_user_tasks/signals.py b/cms/djangoapps/cms_user_tasks/signals.py new file mode 100644 index 000000000000..6640ab2a18df --- /dev/null +++ b/cms/djangoapps/cms_user_tasks/signals.py @@ -0,0 +1,55 @@ +""" +Receivers of signals sent from django-user-tasks +""" +from __future__ import absolute_import, print_function, unicode_literals + +import logging + +from django.core.urlresolvers import reverse +from django.dispatch import receiver +from user_tasks.models import UserTaskArtifact +from user_tasks.signals import user_task_stopped + +from six.moves.urllib.parse import urljoin # pylint: disable=import-error + +from .tasks import send_task_complete_email + +LOGGER = logging.getLogger(__name__) + + +@receiver(user_task_stopped, dispatch_uid="cms_user_task_stopped") +def user_task_stopped_handler(sender, **kwargs): # pylint: disable=unused-argument + """ + Handles sending notifications when a django-user-tasks completes. + This is a signal receiver for user_task_stopped. Currently it only sends + a generic "task completed" email, and only when a top-level task + completes. Eventually it might make more sense to create specific per-task + handlers. + Arguments: + sender (obj): Currently the UserTaskStatus object class + **kwargs: See below + Keywork Arguments: + status (obj): UserTaskStatus of the completed task + Returns: + None + """ + status = kwargs['status'] + + # Only send email when the entire task is complete, should only send when + # a chain / chord / etc completes, not on sub-tasks. + if status.parent is None: + # `name` and `status` are not unique, first is our best guess + artifact = UserTaskArtifact.objects.filter(status=status, name="BASE_URL").first() + + detail_url = None + if artifact and artifact.url.startswith(('http://', 'https://')): + detail_url = urljoin( + artifact.url, + reverse('usertaskstatus-detail', args=[status.uuid]) + ) + + try: + # Need to str state_text here because it is a proxy object and won't serialize correctly + send_task_complete_email.delay(status.name.lower(), str(status.state_text), status.user.email, detail_url) + except Exception: # pylint: disable=broad-except + LOGGER.exception("Unable to queue send_task_complete_email") diff --git a/cms/djangoapps/cms_user_tasks/tasks.py b/cms/djangoapps/cms_user_tasks/tasks.py new file mode 100644 index 000000000000..ff27715a7622 --- /dev/null +++ b/cms/djangoapps/cms_user_tasks/tasks.py @@ -0,0 +1,68 @@ +""" +Celery tasks used by cms_user_tasks +""" + +from boto.exception import NoAuthHandlerFound +from celery.exceptions import MaxRetriesExceededError +from celery.task import task +from celery.utils.log import get_task_logger +from django.conf import settings +from django.core import mail + +from edxmako.shortcuts import render_to_string +from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers + +LOGGER = get_task_logger(__name__) +TASK_COMPLETE_EMAIL_MAX_RETRIES = 3 +TASK_COMPLETE_EMAIL_TIMEOUT = 60 + + +@task(bind=True) +def send_task_complete_email(self, task_name, task_state_text, dest_addr, detail_url): + """ + Sending an email to the users when an async task completes. + """ + retries = self.request.retries + + context = { + 'task_name': task_name, + 'task_status': task_state_text, + 'detail_url': detail_url + } + + subject = render_to_string('emails/user_task_complete_email_subject.txt', context) + # Eliminate any newlines + subject = ''.join(subject.splitlines()) + message = render_to_string('emails/user_task_complete_email.txt', context) + + from_address = configuration_helpers.get_value( + 'email_from_address', + settings.DEFAULT_FROM_EMAIL + ) + + try: + mail.send_mail(subject, message, from_address, [dest_addr], fail_silently=False) + LOGGER.info("Task complete email has been sent to User %s", dest_addr) + except NoAuthHandlerFound: + LOGGER.info( + 'Retrying sending email to user %s, attempt # %s of %s', + dest_addr, + retries, + TASK_COMPLETE_EMAIL_MAX_RETRIES + ) + try: + self.retry(countdown=TASK_COMPLETE_EMAIL_TIMEOUT, max_retries=TASK_COMPLETE_EMAIL_MAX_RETRIES) + except MaxRetriesExceededError: + LOGGER.error( + 'Unable to send task completion email to user from "%s" to "%s"', + from_address, + dest_addr, + exc_info=True + ) + except Exception: # pylint: disable=broad-except + LOGGER.exception( + 'Unable to send task completion email to user from "%s" to "%s"', + from_address, + dest_addr, + exc_info=True + ) diff --git a/cms/djangoapps/cms_user_tasks/tests.py b/cms/djangoapps/cms_user_tasks/tests.py new file mode 100644 index 000000000000..4672fc4915f5 --- /dev/null +++ b/cms/djangoapps/cms_user_tasks/tests.py @@ -0,0 +1,238 @@ +""" +Unit tests for integration of the django-user-tasks app and its REST API. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import logging +from uuid import uuid4 + +import mock +from boto.exception import NoAuthHandlerFound +from django.conf import settings +from django.contrib.auth.models import User +from django.core import mail +from django.core.urlresolvers import reverse +from django.test import override_settings +from rest_framework.test import APITestCase +from user_tasks.models import UserTaskArtifact, UserTaskStatus +from user_tasks.serializers import ArtifactSerializer, StatusSerializer + +from .signals import user_task_stopped + + +class MockLoggingHandler(logging.Handler): + """ + Mock logging handler to help check for logging statements + """ + def __init__(self, *args, **kwargs): + self.reset() + logging.Handler.__init__(self, *args, **kwargs) + + def emit(self, record): + """ + Override to catch messages and store them messages in our internal dicts + """ + self.messages[record.levelname.lower()].append(record.getMessage()) + + def reset(self): + """ + Clear out all messages, also called to initially populate messages dict + """ + self.messages = { + 'debug': [], + 'info': [], + 'warning': [], + 'error': [], + 'critical': [], + } + + +# Helper functions for stuff that pylint complains about without disable comments +def _context(response): + """ + Get a context dictionary for a serializer appropriate for the given response. + """ + return {'request': response.wsgi_request} # pylint: disable=no-member + + +def _data(response): + """ + Get the serialized data dictionary from the given REST API test response. + """ + return response.data # pylint: disable=no-member + + +@override_settings(BROKER_URL='memory://localhost/') +class TestUserTasks(APITestCase): + """ + Tests of the django-user-tasks REST API endpoints. + + Detailed tests of the default authorization rules are in the django-user-tasks code. + These tests just verify that the API is exposed and functioning. + """ + + @classmethod + def setUpTestData(cls): + cls.user = User.objects.create_user('test_user', 'test@example.com', 'password') + cls.status = UserTaskStatus.objects.create( + user=cls.user, task_id=str(uuid4()), task_class='test_rest_api.sample_task', name='SampleTask 2', + total_steps=5) + cls.artifact = UserTaskArtifact.objects.create(status=cls.status, text='Lorem ipsum') + + def setUp(self): + super(TestUserTasks, self).setUp() + self.status.refresh_from_db() + self.client.force_authenticate(self.user) # pylint: disable=no-member + + def test_artifact_detail(self): + """ + Users should be able to access artifacts for tasks they triggered. + """ + response = self.client.get(reverse('usertaskartifact-detail', args=[self.artifact.uuid])) + assert response.status_code == 200 + serializer = ArtifactSerializer(self.artifact, context=_context(response)) + assert _data(response) == serializer.data + + def test_artifact_list(self): + """ + Users should be able to access a list of their tasks' artifacts. + """ + response = self.client.get(reverse('usertaskartifact-list')) + assert response.status_code == 200 + serializer = ArtifactSerializer(self.artifact, context=_context(response)) + assert _data(response)['results'] == [serializer.data] + + def test_status_cancel(self): + """ + Users should be able to cancel tasks they no longer wish to complete. + """ + response = self.client.post(reverse('usertaskstatus-cancel', args=[self.status.uuid])) + assert response.status_code == 200 + self.status.refresh_from_db() + assert self.status.state == UserTaskStatus.CANCELED + + def test_status_delete(self): + """ + Users should be able to delete their own task status records when they're done with them. + """ + response = self.client.delete(reverse('usertaskstatus-detail', args=[self.status.uuid])) + assert response.status_code == 204 + assert not UserTaskStatus.objects.filter(pk=self.status.id).exists() + + def test_status_detail(self): + """ + Users should be able to access status records for tasks they triggered. + """ + response = self.client.get(reverse('usertaskstatus-detail', args=[self.status.uuid])) + assert response.status_code == 200 + serializer = StatusSerializer(self.status, context=_context(response)) + assert _data(response) == serializer.data + + def test_status_list(self): + """ + Users should be able to access a list of their tasks' status records. + """ + response = self.client.get(reverse('usertaskstatus-list')) + assert response.status_code == 200 + serializer = StatusSerializer([self.status], context=_context(response), many=True) + assert _data(response)['results'] == serializer.data + + +@override_settings(BROKER_URL='memory://localhost/') +class TestUserTaskStopped(APITestCase): + """ + Tests of the django-user-tasks signal handling and email integration. + """ + + @classmethod + def setUpTestData(cls): + cls.user = User.objects.create_user('test_user', 'test@example.com', 'password') + cls.status = UserTaskStatus.objects.create( + user=cls.user, task_id=str(uuid4()), task_class='test_rest_api.sample_task', name='SampleTask 2', + total_steps=5) + + def setUp(self): + super(TestUserTaskStopped, self).setUp() + self.status.refresh_from_db() + self.client.force_authenticate(self.user) # pylint: disable=no-member + + def test_email_sent_with_site(self): + """ + Check the signal receiver and email sending. + """ + UserTaskArtifact.objects.create( + status=self.status, name='BASE_URL', url='https://test.edx.org/' + ) + user_task_stopped.send(sender=UserTaskStatus, status=self.status) + + subject = "{platform_name} {studio_name}: Task Status Update".format( + platform_name=settings.PLATFORM_NAME, studio_name=settings.STUDIO_NAME + ) + body_fragments = [ + "Your {task_name} task has completed with the status".format(task_name=self.status.name.lower()), + "https://test.edx.org/", + reverse('usertaskstatus-detail', args=[self.status.uuid]) + ] + + self.assertEqual(len(mail.outbox), 1) + + msg = mail.outbox[0] + + self.assertEqual(msg.subject, subject) + for fragment in body_fragments: + self.assertIn(fragment, msg.body) + + def test_email_not_sent_for_child(self): + """ + No email should be send for child tasks in chords, chains, etc. + """ + child_status = UserTaskStatus.objects.create( + user=self.user, task_id=str(uuid4()), task_class='test_rest_api.sample_task', name='SampleTask 2', + total_steps=5, parent=self.status) + user_task_stopped.send(sender=UserTaskStatus, status=child_status) + self.assertEqual(len(mail.outbox), 0) + + def test_email_sent_without_site(self): + """ + Make sure we send a generic email if the BASE_URL artifact doesn't exist + """ + user_task_stopped.send(sender=UserTaskStatus, status=self.status) + + subject = "{platform_name} {studio_name}: Task Status Update".format( + platform_name=settings.PLATFORM_NAME, studio_name=settings.STUDIO_NAME + ) + fragments = [ + "Your {task_name} task has completed with the status".format(task_name=self.status.name.lower()), + "Sign in to view the details of your task or download any files created." + ] + + self.assertEqual(len(mail.outbox), 1) + + msg = mail.outbox[0] + self.assertEqual(msg.subject, subject) + + for fragment in fragments: + self.assertIn(fragment, msg.body) + + def test_email_retries(self): + """ + Make sure we can succeed on retries + """ + with mock.patch('django.core.mail.send_mail') as mock_exception: + mock_exception.side_effect = NoAuthHandlerFound() + + with mock.patch('cms_user_tasks.tasks.send_task_complete_email.retry') as mock_retry: + user_task_stopped.send(sender=UserTaskStatus, status=self.status) + self.assertTrue(mock_retry.called) + + def test_queue_email_failure(self): + logger = logging.getLogger("cms_user_tasks.signals") + hdlr = MockLoggingHandler(level="DEBUG") + logger.addHandler(hdlr) + + with mock.patch('cms_user_tasks.tasks.send_task_complete_email.delay') as mock_delay: + mock_delay.side_effect = NoAuthHandlerFound() + user_task_stopped.send(sender=UserTaskStatus, status=self.status) + self.assertTrue(mock_delay.called) + self.assertEqual(hdlr.messages['error'][0], u'Unable to queue send_task_complete_email') diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py index 21904959f8e6..c28a5355e87d 100644 --- a/cms/djangoapps/contentstore/admin.py +++ b/cms/djangoapps/contentstore/admin.py @@ -2,10 +2,10 @@ Admin site bindings for contentstore """ +from config_models.admin import ConfigurationModelAdmin from django.contrib import admin -from config_models.admin import ConfigurationModelAdmin -from contentstore.models import VideoUploadConfig, PushNotificationConfig +from contentstore.models import PushNotificationConfig, VideoUploadConfig admin.site.register(VideoUploadConfig, ConfigurationModelAdmin) admin.site.register(PushNotificationConfig, ConfigurationModelAdmin) diff --git a/docs/en_us/enrollment_api/source/__init__.py b/cms/djangoapps/contentstore/api/__init__.py similarity index 100% rename from docs/en_us/enrollment_api/source/__init__.py rename to cms/djangoapps/contentstore/api/__init__.py diff --git a/cms/djangoapps/contentstore/api/tests/test_views.py b/cms/djangoapps/contentstore/api/tests/test_views.py new file mode 100644 index 000000000000..20a49a9751a3 --- /dev/null +++ b/cms/djangoapps/contentstore/api/tests/test_views.py @@ -0,0 +1,188 @@ +""" +Tests for the course import API views +""" +import os +import shutil +import tarfile +import tempfile +from datetime import datetime +from urllib import urlencode + +from django.core.urlresolvers import reverse +from path import Path as path +from mock import patch +from rest_framework import status +from rest_framework.test import APITestCase + +from lms.djangoapps.courseware.tests.factories import GlobalStaffFactory, StaffFactory +from student.tests.factories import UserFactory +from user_tasks.models import UserTaskStatus +from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, SharedModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory + + +class CourseImportViewTest(SharedModuleStoreTestCase, APITestCase): + """ + Test importing courses via a RESTful API (POST method only) + """ + MODULESTORE = TEST_DATA_SPLIT_MODULESTORE + + @classmethod + def setUpClass(cls): + super(CourseImportViewTest, cls).setUpClass() + + cls.course = CourseFactory.create(display_name='test course', run="Testing_course") + cls.course_key = cls.course.id + + cls.restricted_course = CourseFactory.create(display_name='restricted test course', run="Restricted_course") + cls.restricted_course_key = cls.restricted_course.id + + cls.password = 'test' + cls.student = UserFactory(username='dummy', password=cls.password) + cls.staff = StaffFactory(course_key=cls.course.id, password=cls.password) + cls.restricted_staff = StaffFactory(course_key=cls.restricted_course.id, password=cls.password) + + cls.content_dir = path(tempfile.mkdtemp()) + + # Create tar test files ----------------------------------------------- + # OK course: + good_dir = tempfile.mkdtemp(dir=cls.content_dir) + # test course being deeper down than top of tar file + embedded_dir = os.path.join(good_dir, "grandparent", "parent") + os.makedirs(os.path.join(embedded_dir, "course")) + with open(os.path.join(embedded_dir, "course.xml"), "w+") as f: + f.write('') + + with open(os.path.join(embedded_dir, "course", "2013_Spring.xml"), "w+") as f: + f.write('') + + cls.good_tar_filename = "good.tar.gz" + cls.good_tar_fullpath = os.path.join(cls.content_dir, cls.good_tar_filename) + with tarfile.open(cls.good_tar_fullpath, "w:gz") as gtar: + gtar.add(good_dir) + + def get_url(self, course_id): + """ + Helper function to create the url + """ + return reverse( + 'courses_api:course_import', + kwargs={ + 'course_id': course_id + } + ) + + def test_anonymous_import_fails(self): + """ + Test that an anonymous user cannot access the API and an error is received. + """ + with open(self.good_tar_fullpath, 'rb') as fp: + resp = self.client.post(self.get_url(self.course_key), {'course_data': fp}, format='multipart') + self.assertEqual(resp.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_student_import_fails(self): + """ + Test that a student user cannot access the API and an error is received. + """ + self.client.login(username=self.student.username, password=self.password) + with open(self.good_tar_fullpath, 'rb') as fp: + resp = self.client.post(self.get_url(self.course_key), {'course_data': fp}, format='multipart') + self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) + + def test_staff_with_access_import_succeeds(self): + """ + Test that a staff user can access the API and successfully upload a course + """ + self.client.login(username=self.staff.username, password=self.password) + with open(self.good_tar_fullpath, 'rb') as fp: + resp = self.client.post(self.get_url(self.course_key), {'course_data': fp}, format='multipart') + self.assertEqual(resp.status_code, status.HTTP_200_OK) + + def test_staff_has_no_access_import_fails(self): + """ + Test that a staff user can't access another course via the API + """ + self.client.login(username=self.staff.username, password=self.password) + with open(self.good_tar_fullpath, 'rb') as fp: + resp = self.client.post(self.get_url(self.restricted_course_key), {'course_data': fp}, format='multipart') + self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) + + def test_student_get_status_fails(self): + """ + Test that a student user cannot access the API and an error is received. + """ + self.client.login(username=self.student.username, password=self.password) + resp = self.client.get(self.get_url(self.course_key), {'task_id': '1234', 'filename': self.good_tar_filename}) + self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) + + def test_anonymous_get_status_fails(self): + """ + Test that an anonymous user cannot access the API and an error is received. + """ + resp = self.client.get(self.get_url(self.course_key), {'task_id': '1234', 'filename': self.good_tar_filename}) + self.assertEqual(resp.status_code, status.HTTP_401_UNAUTHORIZED) + + def test_staff_get_status_succeeds(self): + """ + Test that an import followed by a get status results in success + + Note: This relies on the fact that we process imports synchronously during testing + """ + self.client.login(username=self.staff.username, password=self.password) + with open(self.good_tar_fullpath, 'rb') as fp: + resp = self.client.post(self.get_url(self.course_key), {'course_data': fp}, format='multipart') + self.assertEqual(resp.status_code, status.HTTP_200_OK) + resp = self.client.get( + self.get_url(self.course_key), + {'task_id': resp.data['task_id'], 'filename': self.good_tar_filename}, + format='multipart' + ) + self.assertEqual(resp.data['state'], UserTaskStatus.SUCCEEDED) + + def test_staff_no_access_get_status_fails(self): + """ + Test that an import followed by a get status as an unauthorized staff fails + + Note: This relies on the fact that we process imports synchronously during testing + """ + self.client.login(username=self.staff.username, password=self.password) + with open(self.good_tar_fullpath, 'rb') as fp: + resp = self.client.post(self.get_url(self.course_key), {'course_data': fp}, format='multipart') + self.assertEqual(resp.status_code, status.HTTP_200_OK) + + task_id = resp.data['task_id'] + resp = self.client.get( + self.get_url(self.course_key), + {'task_id': task_id, 'filename': self.good_tar_filename}, + format='multipart' + ) + self.assertEqual(resp.data['state'], UserTaskStatus.SUCCEEDED) + + self.client.logout() + + self.client.login(username=self.restricted_staff.username, password=self.password) + resp = self.client.get( + self.get_url(self.course_key), + {'task_id': task_id, 'filename': self.good_tar_filename}, + format='multipart' + ) + self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) + + def test_course_task_mismatch_get_status_fails(self): + """ + Test that an import followed by a get status as an unauthorized staff fails + + Note: This relies on the fact that we process imports synchronously during testing + """ + self.client.login(username=self.staff.username, password=self.password) + with open(self.good_tar_fullpath, 'rb') as fp: + resp = self.client.post(self.get_url(self.course_key), {'course_data': fp}, format='multipart') + self.assertEqual(resp.status_code, status.HTTP_200_OK) + + task_id = resp.data['task_id'] + resp = self.client.get( + self.get_url(self.restricted_course_key), + {'task_id': task_id, 'filename': self.good_tar_filename}, + format='multipart' + ) + self.assertEqual(resp.status_code, status.HTTP_403_FORBIDDEN) diff --git a/cms/djangoapps/contentstore/api/urls.py b/cms/djangoapps/contentstore/api/urls.py new file mode 100644 index 000000000000..39dcc8319d77 --- /dev/null +++ b/cms/djangoapps/contentstore/api/urls.py @@ -0,0 +1,18 @@ +""" Course Import API URLs. """ +from django.conf import settings +from django.conf.urls import ( + patterns, + url, +) + +from cms.djangoapps.contentstore.api import views + +urlpatterns = patterns( + '', + url( + r'^v0/import/{course_id}/$'.format( + course_id=settings.COURSE_ID_PATTERN, + ), + views.CourseImportView.as_view(), name='course_import' + ), +) diff --git a/cms/djangoapps/contentstore/api/views.py b/cms/djangoapps/contentstore/api/views.py new file mode 100644 index 000000000000..1e0ad2050a48 --- /dev/null +++ b/cms/djangoapps/contentstore/api/views.py @@ -0,0 +1,190 @@ +""" API v0 views. """ +import base64 +import logging +import os + +from path import Path as path +from six import text_type + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.views.decorators.csrf import csrf_exempt + +from django.core.files import File +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from rest_framework import status +from rest_framework.exceptions import AuthenticationFailed +from rest_framework.generics import GenericAPIView +from rest_framework.response import Response +from user_tasks.models import UserTaskArtifact, UserTaskStatus + +from student.auth import has_course_author_access + +from contentstore.storage import course_import_export_storage +from contentstore.tasks import CourseImportTask, import_olx +from openedx.core.lib.api.view_utils import DeveloperErrorViewMixin, view_auth_classes + +log = logging.getLogger(__name__) + + +@view_auth_classes() +class CourseImportExportViewMixin(DeveloperErrorViewMixin): + """ + Mixin class for course import/export related views. + """ + def perform_authentication(self, request): + """ + Ensures that the user is authenticated (e.g. not an AnonymousUser) + """ + super(CourseImportExportViewMixin, self).perform_authentication(request) + if request.user.is_anonymous(): + raise AuthenticationFailed + + +class CourseImportView(CourseImportExportViewMixin, GenericAPIView): + """ + **Use Case** + + * Start an asynchronous task to import a course from a .tar.gz file into + the specified course ID, overwriting the existing course + * Get a status on an asynchronous task import + + **Example Requests** + + POST /api/courses/v0/import/{course_id}/ + GET /api/courses/v0/import/{course_id}/?task_id={task_id} + + **POST Parameters** + + A POST request must include the following parameters. + + * course_id: (required) A string representation of a Course ID, + e.g., course-v1:edX+DemoX+Demo_Course + * course_data: (required) The course .tar.gz file to import + + **POST Response Values** + + If the import task is started successfully, an HTTP 200 "OK" response is + returned. + + The HTTP 200 response has the following values. + + * task_id: UUID of the created task, usable for checking status + * filename: string of the uploaded filename + + + **Example POST Response** + + { + "task_id": "4b357bb3-2a1e-441d-9f6c-2210cf76606f" + } + + **GET Parameters** + + A GET request must include the following parameters. + + * task_id: (required) The UUID of the task to check, e.g. "4b357bb3-2a1e-441d-9f6c-2210cf76606f" + * filename: (required) The filename of the uploaded course .tar.gz + + **GET Response Values** + + If the import task is found successfully by the UUID provided, an HTTP + 200 "OK" response is returned. + + The HTTP 200 response has the following values. + + * state: String description of the state of the task + + + **Example GET Response** + + { + "state": "Succeeded" + } + + """ + def post(self, request, course_id): + """ + Kicks off an asynchronous course import and returns an ID to be used to check + the task's status + """ + + courselike_key = CourseKey.from_string(course_id) + if not has_course_author_access(request.user, courselike_key): + return self.make_error_response( + status_code=status.HTTP_403_FORBIDDEN, + developer_message='The user requested does not have the required permissions.', + error_code='user_mismatch' + ) + try: + if 'course_data' not in request.FILES: + return self.make_error_response( + status_code=status.HTTP_400_BAD_REQUEST, + developer_message='Missing required parameter', + error_code='internal_error', + field_errors={'course_data': '"course_data" parameter is required, and must be a .tar.gz file'} + ) + + filename = request.FILES['course_data'].name + if not filename.endswith('.tar.gz'): + return self.make_error_response( + status_code=status.HTTP_400_BAD_REQUEST, + developer_message='Parameter in the wrong format', + error_code='internal_error', + field_errors={'course_data': '"course_data" parameter is required, and must be a .tar.gz file'} + ) + course_dir = path(settings.GITHUB_REPO_ROOT) / base64.urlsafe_b64encode(repr(courselike_key)) + temp_filepath = course_dir / filename + if not course_dir.isdir(): # pylint: disable=no-value-for-parameter + os.mkdir(course_dir) + + log.debug('importing course to {0}'.format(temp_filepath)) + with open(temp_filepath, "wb+") as temp_file: + for chunk in request.FILES['course_data'].chunks(): + temp_file.write(chunk) + + log.info("Course import %s: Upload complete", courselike_key) + with open(temp_filepath, 'rb') as local_file: + django_file = File(local_file) + storage_path = course_import_export_storage.save(u'olx_import/' + filename, django_file) + + async_result = import_olx.delay( + request.user.id, text_type(courselike_key), storage_path, filename, request.LANGUAGE_CODE) + return Response({ + 'task_id': async_result.task_id + }) + except Exception as e: + return self.make_error_response( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + developer_message=str(e), + error_code='internal_error' + ) + + def get(self, request, course_id): + """ + Check the status of the specified task + """ + + courselike_key = CourseKey.from_string(course_id) + if not has_course_author_access(request.user, courselike_key): + return self.make_error_response( + status_code=status.HTTP_403_FORBIDDEN, + developer_message='The user requested does not have the required permissions.', + error_code='user_mismatch' + ) + try: + task_id = request.GET['task_id'] + filename = request.GET['filename'] + args = {u'course_key_string': course_id, u'archive_name': filename} + name = CourseImportTask.generate_name(args) + task_status = UserTaskStatus.objects.filter(name=name, task_id=task_id).first() + return Response({ + 'state': task_status.state + }) + except Exception as e: + return self.make_error_response( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + developer_message=str(e), + error_code='internal_error' + ) diff --git a/cms/djangoapps/contentstore/apps.py b/cms/djangoapps/contentstore/apps.py new file mode 100644 index 000000000000..770ce3a9a092 --- /dev/null +++ b/cms/djangoapps/contentstore/apps.py @@ -0,0 +1,22 @@ +""" +Contentstore Application Configuration + +Above-modulestore level signal handlers are connected here. +""" + +from django.apps import AppConfig + + +class ContentstoreConfig(AppConfig): + """ + Application Configuration for Contentstore. + """ + name = u'contentstore' + + def ready(self): + """ + Connect handlers to signals. + """ + # Can't import models at module level in AppConfigs, and models get + # included from the signal handlers + from .signals import handlers # pylint: disable=unused-variable diff --git a/cms/djangoapps/contentstore/context_processors.py b/cms/djangoapps/contentstore/context_processors.py deleted file mode 100644 index 082e52c65410..000000000000 --- a/cms/djangoapps/contentstore/context_processors.py +++ /dev/null @@ -1,28 +0,0 @@ -""" -Django Template Context Processor for CMS Online Contextual Help -""" -import ConfigParser -from django.conf import settings - -from util.help_context_processor import common_doc_url - - -# Open and parse the configuration file when the module is initialized -CONFIG_FILE = open(settings.REPO_ROOT / "docs" / "cms_config.ini") -CONFIG = ConfigParser.ConfigParser() -CONFIG.readfp(CONFIG_FILE) - - -def doc_url(request=None): # pylint: disable=unused-argument - """ - This function is added in the list of TEMPLATES 'context_processors' OPTION, which is a django setting for - a tuple of callables that take a request object as their argument and return a dictionary of items - to be merged into the RequestContext. - - This function returns a dict with get_online_help_info, making it directly available to all mako templates. - - Args: - request: Currently not used, but is passed by django to context processors. - May be used in the future for determining the language of choice. - """ - return common_doc_url(request, CONFIG) diff --git a/cms/djangoapps/contentstore/course_group_config.py b/cms/djangoapps/contentstore/course_group_config.py index dcbf19b30ee2..9e8285a50575 100644 --- a/cms/djangoapps/contentstore/course_group_config.py +++ b/cms/djangoapps/contentstore/course_group_config.py @@ -4,24 +4,26 @@ import json import logging -from util.db import generate_int_id, MYSQL_MAX_INT - from django.utils.translation import ugettext as _ + from contentstore.utils import reverse_usage_url -from xmodule.partitions.partitions import UserPartition -from xmodule.split_test_module import get_split_user_partitions from openedx.core.djangoapps.course_groups.partition_scheme import get_cohorted_user_partition +from util.db import MYSQL_MAX_INT, generate_int_id +from xmodule.partitions.partitions import MINIMUM_STATIC_PARTITION_ID, UserPartition +from xmodule.partitions.partitions_service import get_all_partitions_for_course +from xmodule.split_test_module import get_split_user_partitions -MINIMUM_GROUP_ID = 100 +MINIMUM_GROUP_ID = MINIMUM_STATIC_PARTITION_ID RANDOM_SCHEME = "random" COHORT_SCHEME = "cohort" +ENROLLMENT_SCHEME = "enrollment_track" -# Note: the following content group configuration strings are not -# translated since they are not visible to users. -CONTENT_GROUP_CONFIGURATION_DESCRIPTION = 'The groups in this configuration can be mapped to cohort groups in the LMS.' +CONTENT_GROUP_CONFIGURATION_DESCRIPTION = _( + 'The groups in this configuration can be mapped to cohorts in the Instructor Dashboard.' +) -CONTENT_GROUP_CONFIGURATION_NAME = 'Content Group Configuration' +CONTENT_GROUP_CONFIGURATION_NAME = _('Content Groups') log = logging.getLogger(__name__) @@ -84,7 +86,7 @@ def assign_group_ids(self): """ Assign ids for the group_configuration's groups. """ - used_ids = [g.id for p in self.course.user_partitions for g in p.groups] + used_ids = [g.id for p in get_all_partitions_for_course(self.course) for g in p.groups] # Assign ids to every group in configuration. for group in self.configuration.get('groups', []): if group.get('id') is None: @@ -96,7 +98,7 @@ def get_used_ids(course): """ Return a list of IDs that already in use. """ - return set([p.id for p in course.user_partitions]) + return set([p.id for p in get_all_partitions_for_course(course)]) def get_user_partition(self): """ @@ -186,21 +188,10 @@ def _get_content_experiment_usage_info(store, course, split_tests): # pylint: d return usage_info @staticmethod - def get_content_groups_usage_info(store, course): - """ - Get usage information for content groups. - """ - items = store.get_items(course.id, settings={'group_access': {'$exists': True}}, include_orphans=False) - - return GroupConfiguration._get_content_groups_usage_info(course, items) - - @staticmethod - def _get_content_groups_usage_info(course, items): + def get_partitions_usage_info(store, course): """ Returns all units names and their urls. - This will return only groups for the cohort user partition. - Returns: {'group_id': [ @@ -215,8 +206,10 @@ def _get_content_groups_usage_info(course, items): ], } """ + items = store.get_items(course.id, settings={'group_access': {'$exists': True}}, include_orphans=False) + usage_info = {} - for item, group_id in GroupConfiguration._iterate_items_and_content_group_ids(course, items): + for item, group_id in GroupConfiguration._iterate_items_and_group_ids(course, items): if group_id not in usage_info: usage_info[group_id] = [] @@ -266,7 +259,7 @@ def _get_content_groups_items_usage_info(course, items): } """ usage_info = {} - for item, group_id in GroupConfiguration._iterate_items_and_content_group_ids(course, items): + for item, group_id in GroupConfiguration._iterate_items_and_group_ids(course, items): if group_id not in usage_info: usage_info[group_id] = [] @@ -281,22 +274,23 @@ def _get_content_groups_items_usage_info(course, items): return usage_info @staticmethod - def _iterate_items_and_content_group_ids(course, items): + def _iterate_items_and_group_ids(course, items): """ - Iterate through items and content group IDs in a course. + Iterate through items and group IDs in a course. - This will yield group IDs *only* for cohort user partitions. + This will yield group IDs for all user partitions except those with a scheme of random. Yields: tuple of (item, group_id) """ - content_group_configuration = get_cohorted_user_partition(course) - if content_group_configuration is not None: - for item in items: - if hasattr(item, 'group_access') and item.group_access: - group_ids = item.group_access.get(content_group_configuration.id, []) + all_partitions = get_all_partitions_for_course(course) + for config in all_partitions: + if config is not None and config.scheme.name != RANDOM_SCHEME: + for item in items: + if hasattr(item, 'group_access') and item.group_access: + group_ids = item.group_access.get(config.id, []) - for group_id in group_ids: - yield item, group_id + for group_id in group_ids: + yield item, group_id @staticmethod def update_usage_info(store, course, configuration): @@ -318,23 +312,23 @@ def update_usage_info(store, course, configuration): configuration_json['usage'] = usage_information.get(configuration.id, []) elif configuration.scheme.name == COHORT_SCHEME: # In case if scheme is "cohort" - configuration_json = GroupConfiguration.update_content_group_usage_info(store, course, configuration) + configuration_json = GroupConfiguration.update_partition_usage_info(store, course, configuration) return configuration_json @staticmethod - def update_content_group_usage_info(store, course, configuration): + def update_partition_usage_info(store, course, configuration): """ - Update usage information for particular Content Group Configuration. + Update usage information for particular Partition Configuration. - Returns json of particular content group configuration updated with usage information. + Returns json of particular partition configuration updated with usage information. """ - usage_info = GroupConfiguration.get_content_groups_usage_info(store, course) - content_group_configuration = configuration.to_json() + usage_info = GroupConfiguration.get_partitions_usage_info(store, course) + partition_configuration = configuration.to_json() - for group in content_group_configuration['groups']: + for group in partition_configuration['groups']: group['usage'] = usage_info.get(group['id'], []) - return content_group_configuration + return partition_configuration @staticmethod def get_or_create_content_group(store, course): @@ -356,9 +350,27 @@ def get_or_create_content_group(store, course): ) return content_group_configuration.to_json() - content_group_configuration = GroupConfiguration.update_content_group_usage_info( + content_group_configuration = GroupConfiguration.update_partition_usage_info( store, course, content_group_configuration ) return content_group_configuration + + @staticmethod + def get_all_user_partition_details(store, course): + """ + Returns all the available partitions with updated usage information + + :return: list of all partitions available with details + """ + all_partitions = get_all_partitions_for_course(course) + all_updated_partitions = [] + for partition in all_partitions: + configuration = GroupConfiguration.update_partition_usage_info( + store, + course, + partition + ) + all_updated_partitions.append(configuration) + return all_updated_partitions diff --git a/cms/djangoapps/contentstore/course_info_model.py b/cms/djangoapps/contentstore/course_info_model.py index c5632b3373d3..d5631d9e947b 100644 --- a/cms/djangoapps/contentstore/course_info_model.py +++ b/cms/djangoapps/contentstore/course_info_model.py @@ -12,18 +12,17 @@ } """ -import re import logging +import re from django.http import HttpResponseBadRequest from django.utils.translation import ugettext as _ -from xmodule.modulestore.exceptions import ItemNotFoundError -from xmodule.modulestore.django import modulestore -from xmodule.html_module import CourseInfoModule - -from openedx.core.lib.xblock_utils import get_course_update_items from cms.djangoapps.contentstore.push_notification import enqueue_push_course_update +from openedx.core.lib.xblock_utils import get_course_update_items +from xmodule.html_module import CourseInfoModule +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import ItemNotFoundError # # This should be in a class which inherits from XmlDescriptor log = logging.getLogger(__name__) diff --git a/cms/djangoapps/contentstore/courseware_index.py b/cms/djangoapps/contentstore/courseware_index.py index 99e0052e2ed4..2b4f14642776 100644 --- a/cms/djangoapps/contentstore/courseware_index.py +++ b/cms/djangoapps/contentstore/courseware_index.py @@ -1,23 +1,25 @@ """ Code to allow module store to interface with courseware index """ from __future__ import absolute_import -from abc import ABCMeta, abstractmethod -from datetime import timedelta + import logging import re -from six import add_metaclass +from abc import ABCMeta, abstractmethod +from datetime import timedelta from django.conf import settings -from django.utils.translation import ugettext_lazy, ugettext as _ from django.core.urlresolvers import resolve +from django.utils.translation import ugettext as _ +from django.utils.translation import ugettext_lazy +from search.search_engine_base import SearchEngine +from six import add_metaclass from contentstore.course_group_config import GroupConfiguration from course_modes.models import CourseMode from eventtracking import tracker from openedx.core.lib.courses import course_image_url -from search.search_engine_base import SearchEngine from xmodule.annotator_mixin import html_to_text -from xmodule.modulestore import ModuleStoreEnum from xmodule.library_tools import normalize_key_for_search +from xmodule.modulestore import ModuleStoreEnum # REINDEX_AGE is the default amount of time that we look back for changes # that might have happened. If we are provided with a time at which the @@ -377,7 +379,7 @@ def do_course_reindex(cls, modulestore, course_key): @classmethod def fetch_group_usage(cls, modulestore, structure): groups_usage_dict = {} - groups_usage_info = GroupConfiguration.get_content_groups_usage_info(modulestore, structure).items() + groups_usage_info = GroupConfiguration.get_partitions_usage_info(modulestore, structure).items() groups_usage_info.extend( GroupConfiguration.get_content_groups_items_usage_info( modulestore, diff --git a/cms/djangoapps/contentstore/debug_file_uploader.py b/cms/djangoapps/contentstore/debug_file_uploader.py index d783e16192b0..12acfe95c398 100644 --- a/cms/djangoapps/contentstore/debug_file_uploader.py +++ b/cms/djangoapps/contentstore/debug_file_uploader.py @@ -1,6 +1,7 @@ -from django.core.files.uploadhandler import FileUploadHandler import time +from django.core.files.uploadhandler import FileUploadHandler + class DebugFileUploader(FileUploadHandler): def __init__(self, request=None): diff --git a/cms/djangoapps/contentstore/features/advanced_settings.py b/cms/djangoapps/contentstore/features/advanced_settings.py index 56cad31ad670..5125fac12f37 100644 --- a/cms/djangoapps/contentstore/features/advanced_settings.py +++ b/cms/djangoapps/contentstore/features/advanced_settings.py @@ -1,9 +1,10 @@ # pylint: disable=missing-docstring # pylint: disable=redefined-outer-name -from lettuce import world, step -from nose.tools import assert_false, assert_equal, assert_regexp_matches -from common import type_in_codemirror, press_the_notification_button, get_codemirror_value +from lettuce import step, world +from nose.tools import assert_equal, assert_false, assert_regexp_matches + +from common import get_codemirror_value, press_the_notification_button, type_in_codemirror KEY_CSS = '.key h3.title' DISPLAY_NAME_KEY = "Course Display Name" diff --git a/cms/djangoapps/contentstore/features/common.py b/cms/djangoapps/contentstore/features/common.py index cb7ad41be013..70ce70d0aedb 100644 --- a/cms/djangoapps/contentstore/features/common.py +++ b/cms/djangoapps/contentstore/features/common.py @@ -2,21 +2,21 @@ # pylint: disable=redefined-outer-name import os -from lettuce import world, step -from nose.tools import assert_true, assert_in -from django.conf import settings - -from student.roles import CourseStaffRole, CourseInstructorRole, GlobalStaff -from student.models import get_user +from logging import getLogger +from django.conf import settings +from lettuce import step, world +from nose.tools import assert_in, assert_true from selenium.webdriver.common.keys import Keys -from logging import getLogger -from student.tests.factories import AdminFactory from student import auth +from student.models import get_user +from student.roles import CourseInstructorRole, CourseStaffRole, GlobalStaff +from student.tests.factories import AdminFactory +from terrain.browser import reset_data + logger = getLogger(__name__) -from terrain.browser import reset_data TEST_ROOT = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/features/component.py b/cms/djangoapps/contentstore/features/component.py index ce470fe81e87..a884f08fa09f 100644 --- a/cms/djangoapps/contentstore/features/component.py +++ b/cms/djangoapps/contentstore/features/component.py @@ -5,8 +5,8 @@ # argument name "step" instead of "_step" and pylint does not like that. # pylint: disable=unused-argument -from lettuce import world, step -from nose.tools import assert_true, assert_in, assert_equal +from lettuce import step, world +from nose.tools import assert_equal, assert_in, assert_true DISPLAY_NAME = "Display Name" diff --git a/cms/djangoapps/contentstore/features/component_settings_editor_helpers.py b/cms/djangoapps/contentstore/features/component_settings_editor_helpers.py index 6f572e9ae614..a8474d70e600 100644 --- a/cms/djangoapps/contentstore/features/component_settings_editor_helpers.py +++ b/cms/djangoapps/contentstore/features/component_settings_editor_helpers.py @@ -3,10 +3,11 @@ from lettuce import world from nose.tools import assert_equal, assert_in -from terrain.steps import reload_the_page -from common import type_in_codemirror from selenium.webdriver.common.keys import Keys +from common import type_in_codemirror +from terrain.steps import reload_the_page + @world.absorb def create_component_instance(step, category, component_type=None, is_advanced=False, advanced_component=None): @@ -129,8 +130,8 @@ def edit_component(index=0): # Verify that the "loading" indication has been hidden. world.wait_for_loading() # Verify that the "edit" button is present. - world.wait_for(lambda _driver: world.css_visible('a.edit-button')) - world.css_click('a.edit-button', index) + world.wait_for(lambda _driver: world.css_visible('.edit-button')) + world.css_click('.edit-button', index) world.wait_for_ajax_complete() diff --git a/cms/djangoapps/contentstore/features/course-settings.py b/cms/djangoapps/contentstore/features/course-settings.py index df44ef9917d1..9df6a5f091c9 100644 --- a/cms/djangoapps/contentstore/features/course-settings.py +++ b/cms/djangoapps/contentstore/features/course-settings.py @@ -1,12 +1,12 @@ # pylint: disable=missing-docstring # pylint: disable=redefined-outer-name -from lettuce import world, step -from selenium.webdriver.common.keys import Keys -from cms.djangoapps.contentstore.features.common import type_in_codemirror from django.conf import settings +from lettuce import step, world +from nose.tools import assert_false, assert_true +from selenium.webdriver.common.keys import Keys -from nose.tools import assert_true, assert_false +from cms.djangoapps.contentstore.features.common import type_in_codemirror TEST_ROOT = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/features/course-updates.py b/cms/djangoapps/contentstore/features/course-updates.py index 3a9d0103c64f..286d44848b8e 100644 --- a/cms/djangoapps/contentstore/features/course-updates.py +++ b/cms/djangoapps/contentstore/features/course-updates.py @@ -1,9 +1,10 @@ # pylint: disable=missing-docstring -from cms.djangoapps.contentstore.features.common import type_in_codemirror, get_codemirror_value -from lettuce import world, step +from lettuce import step, world from nose.tools import assert_in +from cms.djangoapps.contentstore.features.common import get_codemirror_value, type_in_codemirror + @step(u'I go to the course updates page') def go_to_updates(_step): diff --git a/cms/djangoapps/contentstore/features/course_import.py b/cms/djangoapps/contentstore/features/course_import.py index c8293c5d11c3..e75db207bc07 100644 --- a/cms/djangoapps/contentstore/features/course_import.py +++ b/cms/djangoapps/contentstore/features/course_import.py @@ -3,8 +3,9 @@ # pylint: disable=unused-argument import os -from lettuce import world, step + from django.conf import settings +from lettuce import step, world def import_file(filename): diff --git a/cms/djangoapps/contentstore/features/courses.py b/cms/djangoapps/contentstore/features/courses.py index 218faded5844..de94f437ae3e 100644 --- a/cms/djangoapps/contentstore/features/courses.py +++ b/cms/djangoapps/contentstore/features/courses.py @@ -2,9 +2,11 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument -from lettuce import world, step +from lettuce import step, world + from common import * + ############### ACTIONS #################### diff --git a/cms/djangoapps/contentstore/features/discussion-editor.py b/cms/djangoapps/contentstore/features/discussion-editor.py index e7f52df8e62c..804fedcc7e4b 100644 --- a/cms/djangoapps/contentstore/features/discussion-editor.py +++ b/cms/djangoapps/contentstore/features/discussion-editor.py @@ -1,7 +1,7 @@ # disable missing docstring # pylint: disable=missing-docstring -from lettuce import world, step +from lettuce import step, world @step('I have created a Discussion Tag$') diff --git a/cms/djangoapps/contentstore/features/grading.py b/cms/djangoapps/contentstore/features/grading.py index 1ed685538965..c86a9a1cd974 100644 --- a/cms/djangoapps/contentstore/features/grading.py +++ b/cms/djangoapps/contentstore/features/grading.py @@ -1,12 +1,13 @@ # pylint: disable=missing-docstring # pylint: disable=redefined-outer-name -from lettuce import world, step -from common import * -from terrain.steps import reload_the_page +from lettuce import step, world +from nose.tools import assert_equal, assert_in, assert_not_equal from selenium.common.exceptions import InvalidElementStateException + +from common import * from contentstore.utils import reverse_course_url -from nose.tools import assert_in, assert_equal, assert_not_equal +from terrain.steps import reload_the_page @step(u'I am viewing the grading settings') diff --git a/cms/djangoapps/contentstore/features/html-editor.py b/cms/djangoapps/contentstore/features/html-editor.py index 6aebaa4b85ba..a858fabbf055 100644 --- a/cms/djangoapps/contentstore/features/html-editor.py +++ b/cms/djangoapps/contentstore/features/html-editor.py @@ -3,9 +3,10 @@ from collections import OrderedDict -from lettuce import world, step -from nose.tools import assert_in, assert_false, assert_true, assert_equal -from common import type_in_codemirror, get_codemirror_value +from lettuce import step, world +from nose.tools import assert_equal, assert_false, assert_in, assert_true + +from common import get_codemirror_value, type_in_codemirror CODEMIRROR_SELECTOR_PREFIX = "$('iframe').contents().find" diff --git a/cms/djangoapps/contentstore/features/pages.py b/cms/djangoapps/contentstore/features/pages.py index bb3e113b643e..9004a5a0a27c 100644 --- a/cms/djangoapps/contentstore/features/pages.py +++ b/cms/djangoapps/contentstore/features/pages.py @@ -2,10 +2,9 @@ # pylint: disable=redefined-outer-name # pylint: disable=unused-argument -from lettuce import world, step +from lettuce import step, world from nose.tools import assert_equal, assert_in - CSS_FOR_TAB_ELEMENT = "li[data-tab-id='{0}'] input.toggle-checkbox" @@ -38,7 +37,7 @@ def not_see_any_static_pages(step): @step(u'I "(edit|delete)" the static page$') def click_edit_or_delete(step, edit_or_delete): - button_css = 'ul.component-actions a.%s-button' % edit_or_delete + button_css = 'ul.component-actions .%s-button' % edit_or_delete world.css_click(button_css) diff --git a/cms/djangoapps/contentstore/features/problem-editor.py b/cms/djangoapps/contentstore/features/problem-editor.py index b1d7c97159eb..8ff30ef7df0d 100644 --- a/cms/djangoapps/contentstore/features/problem-editor.py +++ b/cms/djangoapps/contentstore/features/problem-editor.py @@ -2,10 +2,12 @@ # pylint: disable=missing-docstring import json -from lettuce import world, step + +from lettuce import step, world from nose.tools import assert_equal, assert_true -from common import type_in_codemirror, open_new_course -from advanced_settings import change_value, ADVANCED_MODULES_KEY + +from advanced_settings import ADVANCED_MODULES_KEY, change_value +from common import open_new_course, type_in_codemirror from course_import import import_file DISPLAY_NAME = "Display Name" diff --git a/cms/djangoapps/contentstore/features/signup.py b/cms/djangoapps/contentstore/features/signup.py index 1a8d9c200259..dfcf29e279ee 100644 --- a/cms/djangoapps/contentstore/features/signup.py +++ b/cms/djangoapps/contentstore/features/signup.py @@ -1,8 +1,8 @@ # pylint: disable=missing-docstring # pylint: disable=redefined-outer-name -from lettuce import world, step -from nose.tools import assert_true, assert_false +from lettuce import step, world +from nose.tools import assert_false, assert_true @step('I fill in the registration form$') diff --git a/cms/djangoapps/contentstore/features/textbooks.py b/cms/djangoapps/contentstore/features/textbooks.py index 9b46dabc5167..3d86eea7ed50 100644 --- a/cms/djangoapps/contentstore/features/textbooks.py +++ b/cms/djangoapps/contentstore/features/textbooks.py @@ -1,10 +1,11 @@ # pylint: disable=missing-docstring -from lettuce import world, step from django.conf import settings -from common import upload_file +from lettuce import step, world from nose.tools import assert_equal +from common import upload_file + TEST_ROOT = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/features/transcripts.py b/cms/djangoapps/contentstore/features/transcripts.py index 87f41b3a9fce..177fe3c12d4c 100644 --- a/cms/djangoapps/contentstore/features/transcripts.py +++ b/cms/djangoapps/contentstore/features/transcripts.py @@ -2,14 +2,14 @@ # pylint: disable=missing-docstring import os -from lettuce import world, step from django.conf import settings +from lettuce import step, world +from splinter.request_handler.request_handler import RequestHandler from xmodule.contentstore.content import StaticContent from xmodule.contentstore.django import contentstore from xmodule.exceptions import NotFoundError -from splinter.request_handler.request_handler import RequestHandler TEST_ROOT = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/features/upload.py b/cms/djangoapps/contentstore/features/upload.py index d46c29eff9d6..dfa7f16601b9 100644 --- a/cms/djangoapps/contentstore/features/upload.py +++ b/cms/djangoapps/contentstore/features/upload.py @@ -1,17 +1,19 @@ # pylint: disable=missing-docstring # pylint: disable=redefined-outer-name -from lettuce import world, step -from lettuce.django import django_url -from django.conf import settings -import requests -import string -import random import os +import random +import string + +import requests +from django.conf import settings from django.contrib.auth.models import User -from student.models import CourseEnrollment +from lettuce import step, world +from lettuce.django import django_url from nose.tools import assert_equal, assert_not_equal +from student.models import CourseEnrollment + TEST_ROOT = settings.COMMON_TEST_DATA_ROOT ASSET_NAMES_CSS = 'td.name-col > span.title > a.filename' diff --git a/cms/djangoapps/contentstore/features/video.py b/cms/djangoapps/contentstore/features/video.py index cf650589bcc7..e8d9e1fe3500 100644 --- a/cms/djangoapps/contentstore/features/video.py +++ b/cms/djangoapps/contentstore/features/video.py @@ -1,6 +1,6 @@ # pylint: disable=missing-docstring -from lettuce import world, step +from lettuce import step, world SELECTORS = { 'spinner': '.video-wrapper .spinner', diff --git a/cms/djangoapps/contentstore/management/commands/clean_cert_name.py b/cms/djangoapps/contentstore/management/commands/clean_cert_name.py index 8914069b7c5b..923e38c63338 100644 --- a/cms/djangoapps/contentstore/management/commands/clean_cert_name.py +++ b/cms/djangoapps/contentstore/management/commands/clean_cert_name.py @@ -7,8 +7,8 @@ from django.core.management.base import BaseCommand -from xmodule.modulestore.django import modulestore from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore Result = namedtuple("Result", ["course_key", "cert_name_short", "cert_name_long", "should_clean"]) diff --git a/cms/djangoapps/contentstore/management/commands/cleanup_assets.py b/cms/djangoapps/contentstore/management/commands/cleanup_assets.py index 7896aca3ec9a..c3524c51c611 100644 --- a/cms/djangoapps/contentstore/management/commands/cleanup_assets.py +++ b/cms/djangoapps/contentstore/management/commands/cleanup_assets.py @@ -5,8 +5,8 @@ import logging from django.core.management.base import BaseCommand -from xmodule.contentstore.django import contentstore +from xmodule.contentstore.django import contentstore log = logging.getLogger(__name__) diff --git a/cms/djangoapps/contentstore/management/commands/clone_course.py b/cms/djangoapps/contentstore/management/commands/clone_course.py index bbd94e52ebc9..2430fb0e8e7d 100644 --- a/cms/djangoapps/contentstore/management/commands/clone_course.py +++ b/cms/djangoapps/contentstore/management/commands/clone_course.py @@ -2,12 +2,13 @@ Script for cloning a course """ from django.core.management.base import BaseCommand, CommandError -from xmodule.modulestore.django import modulestore -from student.roles import CourseInstructorRole, CourseStaffRole -from opaque_keys.edx.keys import CourseKey from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locations import SlashSeparatedCourseKey + +from student.roles import CourseInstructorRole, CourseStaffRole from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore # diff --git a/cms/djangoapps/contentstore/management/commands/create_course.py b/cms/djangoapps/contentstore/management/commands/create_course.py index 19d88a597e61..5908990a099a 100644 --- a/cms/djangoapps/contentstore/management/commands/create_course.py +++ b/cms/djangoapps/contentstore/management/commands/create_course.py @@ -1,11 +1,12 @@ """ Django management command to create a course in a specific modulestore """ -from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User -from xmodule.modulestore import ModuleStoreEnum -from contentstore.views.course import create_new_course_in_store +from django.core.management.base import BaseCommand, CommandError + from contentstore.management.commands.utils import user_from_str +from contentstore.views.course import create_new_course_in_store +from xmodule.modulestore import ModuleStoreEnum class Command(BaseCommand): diff --git a/cms/djangoapps/contentstore/management/commands/delete_course.py b/cms/djangoapps/contentstore/management/commands/delete_course.py index 3f91828f1230..1870bce5bdb7 100644 --- a/cms/djangoapps/contentstore/management/commands/delete_course.py +++ b/cms/djangoapps/contentstore/management/commands/delete_course.py @@ -8,23 +8,49 @@ none """ from django.core.management.base import BaseCommand, CommandError -from .prompt import query_yes_no -from contentstore.utils import delete_course_and_groups -from opaque_keys.edx.keys import CourseKey from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey + +from contentstore.utils import delete_course from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore +from .prompt import query_yes_no + class Command(BaseCommand): """ Delete a MongoDB backed course + Example usage: + $ ./manage.py cms delete_course 'course-v1:edX+DemoX+Demo_Course' --settings=devstack + $ ./manage.py cms delete_course 'course-v1:edX+DemoX+Demo_Course' --keep-instructors --settings=devstack + + Note: + keep-instructors option is added in effort to delete duplicate courses safely. + There happens to be courses with difference of casing in ids, for example + course-v1:DartmouthX+DART.ENGL.01.X+2016_T1 is a duplicate of course-v1:DartmouthX+DART.ENGL.01.x+2016_T1 + (Note the differene in 'x' of course number). These two are independent courses in MongoDB. + Current MYSQL setup is case-insensitive which essentially means there are not + seperate entries (in all course related mysql tables, but here we are concerned about accesses) + for duplicate courses. + This option will make us able to delete course (duplicate one) from + mongo while perserving course's related access data in mysql. """ help = '''Delete a MongoDB backed course''' def add_arguments(self, parser): + """ + Add arguments to the command parser. + """ parser.add_argument('course_key', help="ID of the course to delete.") + parser.add_argument( + '--keep-instructors', + action='store_true', + default=False, + help='Do not remove permissions of users and groups for course', + ) + def handle(self, *args, **options): try: course_key = CourseKey.from_string(options['course_key']) @@ -37,5 +63,5 @@ def handle(self, *args, **options): print 'Going to delete the %s course from DB....' % options['course_key'] if query_yes_no("Deleting course {0}. Confirm?".format(course_key), default="no"): if query_yes_no("Are you sure. This action cannot be undone!", default="no"): - delete_course_and_groups(course_key, ModuleStoreEnum.UserID.mgmt_command) + delete_course(course_key, ModuleStoreEnum.UserID.mgmt_command, options['keep_instructors']) print "Deleted course {}".format(course_key) diff --git a/cms/djangoapps/contentstore/management/commands/delete_orphans.py b/cms/djangoapps/contentstore/management/commands/delete_orphans.py index 48e2689e9f60..c765cda23ac1 100644 --- a/cms/djangoapps/contentstore/management/commands/delete_orphans.py +++ b/cms/djangoapps/contentstore/management/commands/delete_orphans.py @@ -1,8 +1,9 @@ """Script for deleting orphans""" from django.core.management.base import BaseCommand, CommandError -from contentstore.views.item import _delete_orphans -from opaque_keys.edx.keys import CourseKey from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey + +from contentstore.views.item import _delete_orphans from xmodule.modulestore import ModuleStoreEnum diff --git a/cms/djangoapps/contentstore/management/commands/edit_course_tabs.py b/cms/djangoapps/contentstore/management/commands/edit_course_tabs.py index 65d45bf0b206..e63a4f97cb23 100644 --- a/cms/djangoapps/contentstore/management/commands/edit_course_tabs.py +++ b/cms/djangoapps/contentstore/management/commands/edit_course_tabs.py @@ -8,14 +8,14 @@ # from optparse import make_option from django.core.management.base import BaseCommand, CommandError -from .prompt import query_yes_no +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locations import SlashSeparatedCourseKey +from contentstore.views import tabs from courseware.courses import get_course_by_id -from contentstore.views import tabs -from opaque_keys import InvalidKeyError -from opaque_keys.edx.locations import SlashSeparatedCourseKey -from opaque_keys.edx.keys import CourseKey +from .prompt import query_yes_no def print_course(course): diff --git a/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py b/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py index 9466471fbc20..91cc8b3c2428 100644 --- a/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py +++ b/cms/djangoapps/contentstore/management/commands/empty_asset_trashcan.py @@ -1,10 +1,12 @@ from django.core.management.base import BaseCommand, CommandError +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locations import SlashSeparatedCourseKey + from xmodule.contentstore.utils import empty_asset_trashcan from xmodule.modulestore.django import modulestore -from opaque_keys.edx.keys import CourseKey + from .prompt import query_yes_no -from opaque_keys import InvalidKeyError -from opaque_keys.edx.locations import SlashSeparatedCourseKey class Command(BaseCommand): diff --git a/cms/djangoapps/contentstore/management/commands/export.py b/cms/djangoapps/contentstore/management/commands/export.py index 44923c6abf75..7cb621dfae26 100644 --- a/cms/djangoapps/contentstore/management/commands/export.py +++ b/cms/djangoapps/contentstore/management/commands/export.py @@ -4,13 +4,14 @@ import os from django.core.management.base import BaseCommand, CommandError -from xmodule.modulestore.xml_exporter import export_course_to_xml -from xmodule.modulestore.django import modulestore -from opaque_keys.edx.keys import CourseKey -from xmodule.contentstore.django import contentstore from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locations import SlashSeparatedCourseKey +from xmodule.contentstore.django import contentstore +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.xml_exporter import export_course_to_xml + class Command(BaseCommand): """ diff --git a/cms/djangoapps/contentstore/management/commands/export_all_courses.py b/cms/djangoapps/contentstore/management/commands/export_all_courses.py index c5ab34c421f3..54f6b11f03d4 100644 --- a/cms/djangoapps/contentstore/management/commands/export_all_courses.py +++ b/cms/djangoapps/contentstore/management/commands/export_all_courses.py @@ -2,9 +2,10 @@ Script for exporting all courseware from Mongo to a directory and listing the courses which failed to export """ from django.core.management.base import BaseCommand, CommandError -from xmodule.modulestore.xml_exporter import export_course_to_xml -from xmodule.modulestore.django import modulestore + from xmodule.contentstore.django import contentstore +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.xml_exporter import export_course_to_xml class Command(BaseCommand): diff --git a/cms/djangoapps/contentstore/management/commands/export_olx.py b/cms/djangoapps/contentstore/management/commands/export_olx.py index 67a14fcad241..72ac1f1f2690 100644 --- a/cms/djangoapps/contentstore/management/commands/export_olx.py +++ b/cms/djangoapps/contentstore/management/commands/export_olx.py @@ -19,17 +19,16 @@ import re import shutil import tarfile -from tempfile import mktemp, mkdtemp +from tempfile import mkdtemp, mktemp from textwrap import dedent -from path import Path as path - from django.core.management.base import BaseCommand, CommandError +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from path import Path as path from xmodule.modulestore.django import modulestore from xmodule.modulestore.xml_exporter import export_course_to_xml -from opaque_keys import InvalidKeyError -from opaque_keys.edx.keys import CourseKey class Command(BaseCommand): diff --git a/cms/djangoapps/contentstore/management/commands/fix_not_found.py b/cms/djangoapps/contentstore/management/commands/fix_not_found.py index c31c4d446c44..ddf00f9c9754 100644 --- a/cms/djangoapps/contentstore/management/commands/fix_not_found.py +++ b/cms/djangoapps/contentstore/management/commands/fix_not_found.py @@ -3,8 +3,10 @@ """ from django.core.management.base import BaseCommand, CommandError from opaque_keys.edx.keys import CourseKey -from xmodule.modulestore.django import modulestore + from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore + # To run from command line: ./manage.py cms fix_not_found course-v1:org+course+run diff --git a/cms/djangoapps/contentstore/management/commands/force_publish.py b/cms/djangoapps/contentstore/management/commands/force_publish.py index 96f68334b418..6dcea6f95fa8 100644 --- a/cms/djangoapps/contentstore/management/commands/force_publish.py +++ b/cms/djangoapps/contentstore/management/commands/force_publish.py @@ -2,13 +2,16 @@ Script for force publishing a course """ from django.core.management.base import BaseCommand, CommandError -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey + +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore + from .prompt import query_yes_no from .utils import get_course_versions + # To run from command line: ./manage.py cms force_publish course-v1:org+course+run diff --git a/cms/djangoapps/contentstore/management/commands/git_export.py b/cms/djangoapps/contentstore/management/commands/git_export.py index 1edb19534502..cafa7bab3315 100644 --- a/cms/djangoapps/contentstore/management/commands/git_export.py +++ b/cms/djangoapps/contentstore/management/commands/git_export.py @@ -18,12 +18,12 @@ from django.core.management.base import BaseCommand, CommandError from django.utils.translation import ugettext as _ +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locations import SlashSeparatedCourseKey import contentstore.git_export_utils as git_export_utils -from opaque_keys.edx.locations import SlashSeparatedCourseKey -from opaque_keys import InvalidKeyError from contentstore.git_export_utils import GitExportError -from opaque_keys.edx.keys import CourseKey log = logging.getLogger(__name__) diff --git a/cms/djangoapps/contentstore/management/commands/import.py b/cms/djangoapps/contentstore/management/commands/import.py index 9a9483cbc1c6..9cd64670333d 100644 --- a/cms/djangoapps/contentstore/management/commands/import.py +++ b/cms/djangoapps/contentstore/management/commands/import.py @@ -4,12 +4,12 @@ from optparse import make_option from django.core.management.base import BaseCommand, CommandError -from django_comment_common.utils import (seed_permissions_roles, - are_permissions_roles_seeded) -from xmodule.modulestore.xml_importer import import_course_from_xml + +from django_comment_common.utils import are_permissions_roles_seeded, seed_permissions_roles +from xmodule.contentstore.django import contentstore from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore -from xmodule.contentstore.django import contentstore +from xmodule.modulestore.xml_importer import import_course_from_xml class Command(BaseCommand): diff --git a/cms/djangoapps/contentstore/management/commands/migrate_to_split.py b/cms/djangoapps/contentstore/management/commands/migrate_to_split.py index 2a2634f5bc01..3613a26c5e45 100644 --- a/cms/djangoapps/contentstore/management/commands/migrate_to_split.py +++ b/cms/djangoapps/contentstore/management/commands/migrate_to_split.py @@ -2,14 +2,15 @@ Django management command to migrate a course from the old Mongo modulestore to the new split-Mongo modulestore. """ -from django.core.management.base import BaseCommand, CommandError from django.contrib.auth.models import User -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.split_migrator import SplitMigrator -from opaque_keys.edx.keys import CourseKey +from django.core.management.base import BaseCommand, CommandError from opaque_keys import InvalidKeyError -from xmodule.modulestore import ModuleStoreEnum +from opaque_keys.edx.keys import CourseKey + from contentstore.management.commands.utils import user_from_str +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.split_migrator import SplitMigrator class Command(BaseCommand): diff --git a/cms/djangoapps/contentstore/management/commands/populate_creators.py b/cms/djangoapps/contentstore/management/commands/populate_creators.py index f56b3a635e56..fa5a84802891 100644 --- a/cms/djangoapps/contentstore/management/commands/populate_creators.py +++ b/cms/djangoapps/contentstore/management/commands/populate_creators.py @@ -3,13 +3,14 @@ This script is only intended to be run once on a given environment. """ -from course_creators.views import add_user_with_status_granted, add_user_with_status_unrequested -from django.core.management.base import BaseCommand - from django.contrib.auth.models import User +from django.core.management.base import BaseCommand from django.db.utils import IntegrityError + +from course_creators.views import add_user_with_status_granted, add_user_with_status_unrequested from student.roles import CourseInstructorRole, CourseStaffRole + #------------ to run: ./manage.py cms populate_creators --settings=dev diff --git a/cms/djangoapps/contentstore/management/commands/reindex_course.py b/cms/djangoapps/contentstore/management/commands/reindex_course.py index 56cb1008af84..796783ca57bf 100644 --- a/cms/djangoapps/contentstore/management/commands/reindex_course.py +++ b/cms/djangoapps/contentstore/management/commands/reindex_course.py @@ -1,21 +1,20 @@ """ Management command to update courses' search index """ import logging -from django.core.management import BaseCommand, CommandError from optparse import make_option from textwrap import dedent -from contentstore.courseware_index import CoursewareSearchIndexer -from search.search_engine_base import SearchEngine +from django.core.management import BaseCommand, CommandError from elasticsearch import exceptions - -from opaque_keys.edx.keys import CourseKey from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import CourseLocator +from search.search_engine_base import SearchEngine -from .prompt import query_yes_no - +from contentstore.courseware_index import CoursewareSearchIndexer from xmodule.modulestore.django import modulestore +from .prompt import query_yes_no + class Command(BaseCommand): """ diff --git a/cms/djangoapps/contentstore/management/commands/reindex_library.py b/cms/djangoapps/contentstore/management/commands/reindex_library.py index 2c9cabc070f0..99d984f66ef8 100644 --- a/cms/djangoapps/contentstore/management/commands/reindex_library.py +++ b/cms/djangoapps/contentstore/management/commands/reindex_library.py @@ -1,19 +1,18 @@ """ Management command to update libraries' search index """ -from django.core.management import BaseCommand, CommandError from optparse import make_option from textwrap import dedent -from contentstore.courseware_index import LibrarySearchIndexer - -from opaque_keys.edx.keys import CourseKey +from django.core.management import BaseCommand, CommandError from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locations import SlashSeparatedCourseKey from opaque_keys.edx.locator import LibraryLocator -from .prompt import query_yes_no - +from contentstore.courseware_index import LibrarySearchIndexer from xmodule.modulestore.django import modulestore +from .prompt import query_yes_no + class Command(BaseCommand): """ diff --git a/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py b/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py index 7513b8b47cec..fa314ddbd363 100644 --- a/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py +++ b/cms/djangoapps/contentstore/management/commands/restore_asset_from_trashcan.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand, CommandError + from xmodule.contentstore.utils import restore_asset_from_trashcan diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_delete_course.py b/cms/djangoapps/contentstore/management/commands/tests/test_delete_course.py index 5ea41377eb3d..28a26b41ed8d 100644 --- a/cms/djangoapps/contentstore/management/commands/tests/test_delete_course.py +++ b/cms/djangoapps/contentstore/management/commands/tests/test_delete_course.py @@ -6,9 +6,11 @@ from opaque_keys.edx.locations import SlashSeparatedCourseKey from django.core.management import call_command, CommandError +from django.contrib.auth.models import User from contentstore.tests.utils import CourseTestCase from xmodule.modulestore.tests.factories import CourseFactory from xmodule.modulestore.django import modulestore +from student.roles import CourseInstructorRole class DeleteCourseTest(CourseTestCase): @@ -60,3 +62,27 @@ def test_course_deleted(self): patched_yes_no.return_value = True call_command('delete_course', 'TestX/TS01/2015_Q1') self.assertIsNone(modulestore().get_course(SlashSeparatedCourseKey("TestX", "TS01", "2015_Q1"))) + + def test_course_deletion_with_keep_instructors(self): + """ + Tests that deleting course with keep-instructors option do not remove instructors from course. + """ + instructor_user = User.objects.create( + username='test_instructor', + email='test_email@example.com' + ) + self.assertIsNotNone(instructor_user) + + # Add and verify instructor role for the course + instructor_role = CourseInstructorRole(self.course.id) + instructor_role.add_users(instructor_user) + self.assertTrue(instructor_role.has_user(instructor_user)) + + # Verify the course we are about to delete exists in the modulestore + self.assertIsNotNone(modulestore().get_course(self.course.id)) + + with mock.patch(self.YESNO_PATCH_LOCATION, return_value=True): + call_command('delete_course', 'TestX/TS01/2015_Q1', '--keep-instructors') + + self.assertIsNone(modulestore().get_course(self.course.id)) + self.assertTrue(instructor_role.has_user(instructor_user)) diff --git a/cms/djangoapps/contentstore/management/commands/tests/test_export.py b/cms/djangoapps/contentstore/management/commands/tests/test_export.py index dcb8a208aa50..ea753d085cc0 100644 --- a/cms/djangoapps/contentstore/management/commands/tests/test_export.py +++ b/cms/djangoapps/contentstore/management/commands/tests/test_export.py @@ -17,9 +17,6 @@ class TestArgParsingCourseExport(unittest.TestCase): """ Tests for parsing arguments for the `export` management command """ - def setUp(self): - super(TestArgParsingCourseExport, self).setUp() - def test_no_args(self): """ Test export command with no arguments diff --git a/cms/djangoapps/contentstore/management/commands/utils.py b/cms/djangoapps/contentstore/management/commands/utils.py index 03502f844138..0b248ca5262c 100644 --- a/cms/djangoapps/contentstore/management/commands/utils.py +++ b/cms/djangoapps/contentstore/management/commands/utils.py @@ -3,6 +3,7 @@ """ from django.contrib.auth.models import User from opaque_keys.edx.keys import CourseKey + from xmodule.modulestore.django import modulestore diff --git a/cms/djangoapps/contentstore/management/commands/xlint.py b/cms/djangoapps/contentstore/management/commands/xlint.py index 9c08e2de5dec..afb7c73980a3 100644 --- a/cms/djangoapps/contentstore/management/commands/xlint.py +++ b/cms/djangoapps/contentstore/management/commands/xlint.py @@ -2,6 +2,7 @@ Verify the structure of courseware as to it's suitability for import """ from django.core.management.base import BaseCommand, CommandError + from xmodule.modulestore.xml_importer import perform_xlint diff --git a/cms/djangoapps/contentstore/migrations/0001_initial.py b/cms/djangoapps/contentstore/migrations/0001_initial.py index 88126406e34b..ef69433f2a0e 100644 --- a/cms/djangoapps/contentstore/migrations/0001_initial.py +++ b/cms/djangoapps/contentstore/migrations/0001_initial.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from django.db import migrations, models import django.db.models.deletion from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index 12435dd9c26e..7fa8efe4f9da 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -2,9 +2,8 @@ Models for contentstore """ -from django.db.models.fields import TextField - from config_models.models import ConfigurationModel +from django.db.models.fields import TextField class VideoUploadConfig(ConfigurationModel): diff --git a/cms/djangoapps/contentstore/proctoring.py b/cms/djangoapps/contentstore/proctoring.py index 55b31d95c3f3..70d0cbd46e0f 100644 --- a/cms/djangoapps/contentstore/proctoring.py +++ b/cms/djangoapps/contentstore/proctoring.py @@ -5,24 +5,20 @@ import logging from django.conf import settings - -from xmodule.modulestore.django import modulestore - -from contentstore.views.helpers import is_item_in_course_tree - from edx_proctoring.api import ( - get_exam_by_content_id, - update_exam, create_exam, - get_all_exams_for_course, - update_review_policy, create_exam_review_policy, + get_all_exams_for_course, + get_exam_by_content_id, remove_review_policy, + update_exam, + update_review_policy ) -from edx_proctoring.exceptions import ( - ProctoredExamNotFoundException, - ProctoredExamReviewPolicyNotFoundException -) +from edx_proctoring.exceptions import ProctoredExamNotFoundException, ProctoredExamReviewPolicyNotFoundException + +from contentstore.views.helpers import is_item_in_course_tree +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import ItemNotFoundError log = logging.getLogger(__name__) @@ -40,6 +36,9 @@ def register_special_exams(course_key): return course = modulestore().get_course(course_key) + if course is None: + raise ItemNotFoundError("Course {} does not exist", unicode(course_key)) + if not course.enable_proctored_exams and not course.enable_timed_exams: # likewise if course does not have these features turned on # then quickly exit diff --git a/cms/djangoapps/contentstore/push_notification.py b/cms/djangoapps/contentstore/push_notification.py index ec99d6171310..ed038e07dd21 100644 --- a/cms/djangoapps/contentstore/push_notification.py +++ b/cms/djangoapps/contentstore/push_notification.py @@ -2,16 +2,17 @@ Helper methods for push notifications from Studio. """ +from logging import exception as log_exception from uuid import uuid4 + from django.conf import settings -from logging import exception as log_exception -from contentstore.tasks import push_course_update_task from contentstore.models import PushNotificationConfig -from xmodule.modulestore.django import modulestore -from parse_rest.installation import Push +from contentstore.tasks import push_course_update_task from parse_rest.connection import register from parse_rest.core import ParseError +from parse_rest.installation import Push +from xmodule.modulestore.django import modulestore def push_notification_enabled(): diff --git a/docs/en_us/platform_api/__init__.py b/cms/djangoapps/contentstore/signals/__init__.py similarity index 100% rename from docs/en_us/platform_api/__init__.py rename to cms/djangoapps/contentstore/signals/__init__.py diff --git a/cms/djangoapps/contentstore/signals.py b/cms/djangoapps/contentstore/signals/handlers.py similarity index 75% rename from cms/djangoapps/contentstore/signals.py rename to cms/djangoapps/contentstore/signals/handlers.py index 4be065214bb9..6736c03f9bc2 100644 --- a/cms/djangoapps/contentstore/signals.py +++ b/cms/djangoapps/contentstore/signals/handlers.py @@ -2,16 +2,19 @@ import logging from datetime import datetime -from pytz import UTC from django.dispatch import receiver +from pytz import UTC -from xmodule.modulestore.django import modulestore, SignalHandler from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer from contentstore.proctoring import register_special_exams +from lms.djangoapps.grades.tasks import compute_all_grades_for_course from openedx.core.djangoapps.credit.signals import on_course_publish from openedx.core.lib.gating import api as gating_api +from track.event_transaction_utils import get_event_transaction_id, get_event_transaction_type from util.module_utils import yield_dynamic_descriptor_descendants +from .signals import GRADING_POLICY_CHANGED +from xmodule.modulestore.django import SignalHandler, modulestore log = logging.getLogger(__name__) @@ -39,10 +42,9 @@ def listen_for_course_publish(sender, course_key, **kwargs): # pylint: disable= # Finally call into the course search subsystem # to kick off an indexing action - if CoursewareSearchIndexer.indexing_is_enabled(): # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded - from .tasks import update_search_index + from contentstore.tasks import update_search_index update_search_index.delay(unicode(course_key), datetime.now(UTC).isoformat()) @@ -55,7 +57,7 @@ def listen_for_library_update(sender, library_key, **kwargs): # pylint: disable if LibrarySearchIndexer.indexing_is_enabled(): # import here, because signal is registered at startup, but items in tasks are not yet able to be loaded - from .tasks import update_library_index + from contentstore.tasks import update_library_index update_library_index.delay(unicode(library_key), datetime.now(UTC).isoformat()) @@ -85,3 +87,22 @@ def handle_item_deleted(**kwargs): gating_api.remove_prerequisite(module.location) # Remove any 'requires' course content milestone relationships gating_api.set_required_content(course_key, module.location, None, None) + + +@receiver(GRADING_POLICY_CHANGED) +def handle_grading_policy_changed(sender, **kwargs): + # pylint: disable=unused-argument + """ + Receives signal and kicks off celery task to recalculate grades + """ + course_key = kwargs.get('course_key') + result = compute_all_grades_for_course.apply_async( + course_key=course_key, + event_transaction_id=get_event_transaction_id(), + event_transaction_type=get_event_transaction_type(), + ) + log.info("Grades: Created {task_name}[{task_id}] with arguments {kwargs}".format( + task_name=compute_all_grades_for_course.name, + task_id=result.task_id, + kwargs=kwargs, + )) diff --git a/cms/djangoapps/contentstore/signals/signals.py b/cms/djangoapps/contentstore/signals/signals.py new file mode 100644 index 000000000000..87f699416566 --- /dev/null +++ b/cms/djangoapps/contentstore/signals/signals.py @@ -0,0 +1,14 @@ +""" +Contentstore signals +""" +from django.dispatch import Signal + +# Signal that indicates that a course grading policy has been updated. +# This signal is generated when a grading policy change occurs within +# modulestore for either course or subsection changes. +GRADING_POLICY_CHANGED = Signal( + providing_args=[ + 'user_id', # Integer User ID + 'course_id', # Unicode string representing the course + ] +) diff --git a/cms/djangoapps/contentstore/startup.py b/cms/djangoapps/contentstore/startup.py deleted file mode 100644 index a19773bb3791..000000000000 --- a/cms/djangoapps/contentstore/startup.py +++ /dev/null @@ -1,2 +0,0 @@ -""" will register signal handlers, moved out of __init__.py to ensure correct loading order post Django 1.7 """ -from . import signals # pylint: disable=unused-import diff --git a/cms/djangoapps/contentstore/storage.py b/cms/djangoapps/contentstore/storage.py new file mode 100644 index 000000000000..76bd5b54bdc8 --- /dev/null +++ b/cms/djangoapps/contentstore/storage.py @@ -0,0 +1,22 @@ +""" +Storage backend for course import and export. +""" +from __future__ import absolute_import + +from django.conf import settings +from django.core.files.storage import get_storage_class +from storages.backends.s3boto import S3BotoStorage +from storages.utils import setting + + +class ImportExportS3Storage(S3BotoStorage): # pylint: disable=abstract-method + """ + S3 backend for course import and export OLX files. + """ + + def __init__(self): + bucket = setting('COURSE_IMPORT_EXPORT_BUCKET', settings.AWS_STORAGE_BUCKET_NAME) + super(ImportExportS3Storage, self).__init__(bucket=bucket, custom_domain=None, querystring_auth=True) + +# pylint: disable=invalid-name +course_import_export_storage = get_storage_class(settings.COURSE_IMPORT_EXPORT_STORAGE)() diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index b67600e2385d..3061987f80c3 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -1,27 +1,80 @@ """ This file contains celery tasks for contentstore views """ +from __future__ import absolute_import + +import base64 import json -import logging -from celery.task import task -from celery.utils.log import get_task_logger +import os +import shutil +import tarfile from datetime import datetime -from pytz import UTC +from tempfile import NamedTemporaryFile, mkdtemp +from celery.task import task +from celery.utils.log import get_task_logger +from django.conf import settings from django.contrib.auth.models import User +from django.core.exceptions import SuspiciousOperation +from django.core.files import File +from django.test import RequestFactory +from django.utils.text import get_valid_filename +from django.utils.translation import ugettext as _ +from djcelery.common import respect_language +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryLocator +from organizations.models import OrganizationCourse +from path import Path as path +from pytz import UTC +from six import iteritems, text_type +from user_tasks.models import UserTaskArtifact, UserTaskStatus +from user_tasks.tasks import UserTask +import dogstats_wrapper as dog_stats_api from contentstore.courseware_index import CoursewareSearchIndexer, LibrarySearchIndexer, SearchIndexingError -from contentstore.utils import initialize_permissions +from contentstore.storage import course_import_export_storage +from contentstore.utils import initialize_permissions, reverse_usage_url from course_action_state.models import CourseRerunState -from opaque_keys.edx.keys import CourseKey +from models.settings.course_metadata import CourseMetadata +from openedx.core.djangoapps.embargo.models import CountryAccessRule, RestrictedCourse +from openedx.core.lib.extract_tar import safetar_extractall +from student.auth import has_course_author_access +from xmodule.contentstore.django import contentstore from xmodule.course_module import CourseFields +from xmodule.exceptions import SerializationError +from xmodule.modulestore import COURSE_ROOT, LIBRARY_ROOT from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import DuplicateCourseError, ItemNotFoundError +from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml +from xmodule.modulestore.xml_importer import import_course_from_xml, import_library_from_xml LOGGER = get_task_logger(__name__) +FILE_READ_CHUNK = 1024 # bytes FULL_COURSE_REINDEX_THRESHOLD = 1 +def clone_instance(instance, field_values): + """ Clones a Django model instance. + + The specified fields are replaced with new values. + + Arguments: + instance (Model): Instance of a Django model. + field_values (dict): Map of field names to new values. + + Returns: + Model: New instance. + """ + instance.pk = None + + for field, value in iteritems(field_values): + setattr(instance, field, value) + + instance.save() + + return instance + + @task() def rerun_course(source_course_key_string, destination_course_key_string, user_id, fields=None): """ @@ -30,10 +83,10 @@ def rerun_course(source_course_key_string, destination_course_key_string, user_i # import here, at top level this import prevents the celery workers from starting up correctly from edxval.api import copy_course_videos + source_course_key = CourseKey.from_string(source_course_key_string) + destination_course_key = CourseKey.from_string(destination_course_key_string) try: # deserialize the payload - source_course_key = CourseKey.from_string(source_course_key_string) - destination_course_key = CourseKey.from_string(destination_course_key_string) fields = deserialize_fields(fields) if fields else None # use the split modulestore as the store for the rerun course, @@ -51,19 +104,34 @@ def rerun_course(source_course_key_string, destination_course_key_string, user_i # call edxval to attach videos to the rerun copy_course_videos(source_course_key, destination_course_key) + # Copy OrganizationCourse + organization_course = OrganizationCourse.objects.filter(course_id=source_course_key_string).first() + + if organization_course: + clone_instance(organization_course, {'course_id': destination_course_key_string}) + + # Copy RestrictedCourse + restricted_course = RestrictedCourse.objects.filter(course_key=source_course_key).first() + + if restricted_course: + country_access_rules = CountryAccessRule.objects.filter(restricted_course=restricted_course) + new_restricted_course = clone_instance(restricted_course, {'course_key': destination_course_key}) + for country_access_rule in country_access_rules: + clone_instance(country_access_rule, {'restricted_course': new_restricted_course}) + return "succeeded" - except DuplicateCourseError as exc: + except DuplicateCourseError: # do NOT delete the original course, only update the status CourseRerunState.objects.failed(course_key=destination_course_key) - logging.exception(u'Course Rerun Error') + LOGGER.exception(u'Course Rerun Error') return "duplicate course" # catch all exceptions so we can update the state and properly cleanup the course. except Exception as exc: # pylint: disable=broad-except # update state: Failed CourseRerunState.objects.failed(course_key=destination_course_key) - logging.exception(u'Course Rerun Error') + LOGGER.exception(u'Course Rerun Error') try: # cleanup any remnants of the course @@ -72,12 +140,12 @@ def rerun_course(source_course_key_string, destination_course_key_string, user_i # it's possible there was an error even before the course module was created pass - return "exception: " + unicode(exc) + return u"exception: " + text_type(exc) def deserialize_fields(json_fields): fields = json.loads(json_fields) - for field_name, value in fields.iteritems(): + for field_name, value in iteritems(fields): fields[field_name] = getattr(CourseFields, field_name).from_json(value) return fields @@ -99,9 +167,9 @@ def update_search_index(course_id, triggered_time_isoformat): CoursewareSearchIndexer.index(modulestore(), course_key, triggered_at=(_parse_time(triggered_time_isoformat))) except SearchIndexingError as exc: - LOGGER.error('Search indexing error for complete course %s - %s', course_id, unicode(exc)) + LOGGER.error(u'Search indexing error for complete course %s - %s', course_id, text_type(exc)) else: - LOGGER.debug('Search indexing successful for complete course %s', course_id) + LOGGER.debug(u'Search indexing successful for complete course %s', course_id) @task() @@ -112,9 +180,9 @@ def update_library_index(library_id, triggered_time_isoformat): LibrarySearchIndexer.index(modulestore(), library_key, triggered_at=(_parse_time(triggered_time_isoformat))) except SearchIndexingError as exc: - LOGGER.error('Search indexing error for library %s - %s', library_id, unicode(exc)) + LOGGER.error(u'Search indexing error for library %s - %s', library_id, text_type(exc)) else: - LOGGER.debug('Search indexing successful for library %s', library_id) + LOGGER.debug(u'Search indexing successful for library %s', library_id) @task() @@ -125,3 +193,348 @@ def push_course_update_task(course_key_string, course_subscription_id, course_di # TODO Use edx-notifications library instead (MA-638). from .push_notification import send_push_course_update send_push_course_update(course_key_string, course_subscription_id, course_display_name) + + +class CourseExportTask(UserTask): # pylint: disable=abstract-method + """ + Base class for course and library export tasks. + """ + + @staticmethod + def calculate_total_steps(arguments_dict): + """ + Get the number of in-progress steps in the export process, as shown in the UI. + + For reference, these are: + + 1. Exporting + 2. Compressing + """ + return 2 + + @classmethod + def generate_name(cls, arguments_dict): + """ + Create a name for this particular import task instance. + + Arguments: + arguments_dict (dict): The arguments given to the task function + + Returns: + text_type: The generated name + """ + key = arguments_dict[u'course_key_string'] + return u'Export of {}'.format(key) + + +@task(base=CourseExportTask, bind=True) +def export_olx(self, user_id, course_key_string, language): + """ + Export a course or library to an OLX .tar.gz archive and prepare it for download. + """ + courselike_key = CourseKey.from_string(course_key_string) + + try: + user = User.objects.get(pk=user_id) + except User.DoesNotExist: + with respect_language(language): + self.status.fail(_(u'Unknown User ID: {0}').format(user_id)) + return + if not has_course_author_access(user, courselike_key): + with respect_language(language): + self.status.fail(_(u'Permission denied')) + return + + if isinstance(courselike_key, LibraryLocator): + courselike_module = modulestore().get_library(courselike_key) + else: + courselike_module = modulestore().get_course(courselike_key) + + try: + self.status.set_state(u'Exporting') + tarball = create_export_tarball(courselike_module, courselike_key, {}, self.status) + artifact = UserTaskArtifact(status=self.status, name=u'Output') + artifact.file.save(name=tarball.name, content=File(tarball)) # pylint: disable=no-member + artifact.save() + # catch all exceptions so we can record useful error messages + except Exception as exception: # pylint: disable=broad-except + LOGGER.exception(u'Error exporting course %s', courselike_key) + if self.status.state != UserTaskStatus.FAILED: + self.status.fail({'raw_error_msg': text_type(exception)}) + return + + +def create_export_tarball(course_module, course_key, context, status=None): + """ + Generates the export tarball, or returns None if there was an error. + + Updates the context with any error information if applicable. + """ + name = course_module.url_name + export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") + root_dir = path(mkdtemp()) + + try: + if isinstance(course_key, LibraryLocator): + export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name) + else: + export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) + + if status: + status.set_state(u'Compressing') + status.increment_completed_steps() + LOGGER.debug(u'tar file being generated at %s', export_file.name) + with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: + tar_file.add(root_dir / name, arcname=name) + + except SerializationError as exc: + LOGGER.exception(u'There was an error exporting %s', course_key) + parent = None + try: + failed_item = modulestore().get_item(exc.location) + parent_loc = modulestore().get_parent_location(failed_item.location) + + if parent_loc is not None: + parent = modulestore().get_item(parent_loc) + except: # pylint: disable=bare-except + # if we have a nested exception, then we'll show the more generic error message + pass + + context.update({ + 'in_err': True, + 'raw_err_msg': str(exc), + 'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "", + }) + if status: + status.fail(json.dumps({'raw_error_msg': context['raw_err_msg'], + 'edit_unit_url': context['edit_unit_url']})) + raise + except Exception as exc: + LOGGER.exception('There was an error exporting %s', course_key) + context.update({ + 'in_err': True, + 'edit_unit_url': None, + 'raw_err_msg': str(exc)}) + if status: + status.fail(json.dumps({'raw_error_msg': context['raw_err_msg']})) + raise + finally: + if os.path.exists(root_dir / name): + shutil.rmtree(root_dir / name) + + return export_file + + +class CourseImportTask(UserTask): # pylint: disable=abstract-method + """ + Base class for course and library import tasks. + """ + + @staticmethod + def calculate_total_steps(arguments_dict): + """ + Get the number of in-progress steps in the import process, as shown in the UI. + + For reference, these are: + + 1. Unpacking + 2. Verifying + 3. Updating + """ + return 3 + + @classmethod + def generate_name(cls, arguments_dict): + """ + Create a name for this particular import task instance. + + Arguments: + arguments_dict (dict): The arguments given to the task function + + Returns: + text_type: The generated name + """ + key = arguments_dict[u'course_key_string'] + filename = arguments_dict[u'archive_name'] + return u'Import of {} from {}'.format(key, filename) + + +@task(base=CourseImportTask, bind=True) +def import_olx(self, user_id, course_key_string, archive_path, archive_name, language): + """ + Import a course or library from a provided OLX .tar.gz archive. + """ + courselike_key = CourseKey.from_string(course_key_string) + try: + user = User.objects.get(pk=user_id) + except User.DoesNotExist: + with respect_language(language): + self.status.fail(_(u'Unknown User ID: {0}').format(user_id)) + return + if not has_course_author_access(user, courselike_key): + with respect_language(language): + self.status.fail(_(u'Permission denied')) + return + + is_library = isinstance(courselike_key, LibraryLocator) + is_course = not is_library + if is_library: + root_name = LIBRARY_ROOT + courselike_module = modulestore().get_library(courselike_key) + import_func = import_library_from_xml + else: + root_name = COURSE_ROOT + courselike_module = modulestore().get_course(courselike_key) + import_func = import_course_from_xml + + # Locate the uploaded OLX archive (and download it from S3 if necessary) + # Do everything in a try-except block to make sure everything is properly cleaned up. + data_root = path(settings.GITHUB_REPO_ROOT) + subdir = base64.urlsafe_b64encode(repr(courselike_key)) + course_dir = data_root / subdir + try: + self.status.set_state(u'Unpacking') + + if not archive_name.endswith(u'.tar.gz'): + with respect_language(language): + self.status.fail(_(u'We only support uploading a .tar.gz file.')) + return + + temp_filepath = course_dir / get_valid_filename(archive_name) + if not course_dir.isdir(): # pylint: disable=no-value-for-parameter + os.mkdir(course_dir) + + LOGGER.debug(u'importing course to {0}'.format(temp_filepath)) + + # Copy the OLX archive from where it was uploaded to (S3, Swift, file system, etc.) + if not course_import_export_storage.exists(archive_path): + LOGGER.info(u'Course import %s: Uploaded file %s not found', courselike_key, archive_path) + with respect_language(language): + self.status.fail(_(u'Tar file not found')) + return + with course_import_export_storage.open(archive_path, 'rb') as source: + with open(temp_filepath, 'wb') as destination: + def read_chunk(): + """ + Read and return a sequence of bytes from the source file. + """ + return source.read(FILE_READ_CHUNK) + for chunk in iter(read_chunk, b''): + destination.write(chunk) + LOGGER.info(u'Course import %s: Download from storage complete', courselike_key) + # Delete from source location + course_import_export_storage.delete(archive_path) + + # If the course has an entrance exam then remove it and its corresponding milestone. + # current course state before import. + if is_course: + if courselike_module.entrance_exam_enabled: + fake_request = RequestFactory().get(u'/') + fake_request.user = user + from contentstore.views.entrance_exam import remove_entrance_exam_milestone_reference + # TODO: Is this really ok? Seems dangerous for a live course + remove_entrance_exam_milestone_reference(fake_request, courselike_key) + LOGGER.info( + u'entrance exam milestone content reference for course %s has been removed', + courselike_module.id + ) + # Send errors to client with stage at which error occurred. + except Exception as exception: # pylint: disable=broad-except + if course_dir.isdir(): # pylint: disable=no-value-for-parameter + shutil.rmtree(course_dir) + LOGGER.info(u'Course import %s: Temp data cleared', courselike_key) + + LOGGER.exception(u'Error importing course %s', courselike_key) + self.status.fail(text_type(exception)) + return + + # try-finally block for proper clean up after receiving file. + try: + tar_file = tarfile.open(temp_filepath) + try: + safetar_extractall(tar_file, (course_dir + u'/').encode(u'utf-8')) + except SuspiciousOperation as exc: + LOGGER.info(u'Course import %s: Unsafe tar file - %s', courselike_key, exc.args[0]) + with respect_language(language): + self.status.fail(_(u'Unsafe tar file. Aborting import.')) + return + finally: + tar_file.close() + + LOGGER.info(u'Course import %s: Uploaded file extracted', courselike_key) + self.status.set_state(u'Verifying') + self.status.increment_completed_steps() + + # find the 'course.xml' file + def get_all_files(directory): + """ + For each file in the directory, yield a 2-tuple of (file-name, + directory-path) + """ + for directory_path, _dirnames, filenames in os.walk(directory): + for filename in filenames: + yield (filename, directory_path) + + def get_dir_for_filename(directory, filename): + """ + Returns the directory path for the first file found in the directory + with the given name. If there is no file in the directory with + the specified name, return None. + """ + for name, directory_path in get_all_files(directory): + if name == filename: + return directory_path + return None + + dirpath = get_dir_for_filename(course_dir, root_name) + if not dirpath: + with respect_language(language): + self.status.fail(_(u'Could not find the {0} file in the package.').format(root_name)) + return + + dirpath = os.path.relpath(dirpath, data_root) + LOGGER.debug(u'found %s at %s', root_name, dirpath) + + LOGGER.info(u'Course import %s: Extracted file verified', courselike_key) + self.status.set_state(u'Updating') + self.status.increment_completed_steps() + + with dog_stats_api.timer( + u'courselike_import.time', + tags=[u"courselike:{}".format(courselike_key)] + ): + courselike_items = import_func( + modulestore(), user.id, + settings.GITHUB_REPO_ROOT, [dirpath], + load_error_modules=False, + static_content_store=contentstore(), + target_id=courselike_key + ) + + new_location = courselike_items[0].location + LOGGER.debug(u'new course at %s', new_location) + + LOGGER.info(u'Course import %s: Course import successful', courselike_key) + except Exception as exception: # pylint: disable=broad-except + LOGGER.exception(u'error importing course') + self.status.fail(text_type(exception)) + finally: + if course_dir.isdir(): # pylint: disable=no-value-for-parameter + shutil.rmtree(course_dir) + LOGGER.info(u'Course import %s: Temp data cleared', courselike_key) + + if self.status.state == u'Updating' and is_course: + # Reload the course so we have the latest state + course = modulestore().get_course(courselike_key) + if course.entrance_exam_enabled: + entrance_exam_chapter = modulestore().get_items( + course.id, + qualifiers={u'category': u'chapter'}, + settings={u'is_entrance_exam': True} + )[0] + + metadata = {u'entrance_exam_id': text_type(entrance_exam_chapter.location)} + CourseMetadata.update_from_dict(metadata, course, user) + from contentstore.views.entrance_exam import add_entrance_exam_milestone + add_entrance_exam_milestone(course.id, entrance_exam_chapter) + LOGGER.info(u'Course %s Entrance exam imported', course.id) diff --git a/cms/djangoapps/contentstore/tests/test_clone_course.py b/cms/djangoapps/contentstore/tests/test_clone_course.py index a0c9428dd08f..82b931a54f20 100644 --- a/cms/djangoapps/contentstore/tests/test_clone_course.py +++ b/cms/djangoapps/contentstore/tests/test_clone_course.py @@ -2,21 +2,21 @@ Unit tests for cloning a course between the same and different module stores. """ import json -from django.conf import settings +from django.conf import settings +from mock import Mock, patch from opaque_keys.edx.locator import CourseLocator -from xmodule.modulestore import ModuleStoreEnum, EdxJSONEncoder -from contentstore.tests.utils import CourseTestCase + from contentstore.tasks import rerun_course -from student.auth import has_course_author_access -from course_action_state.models import CourseRerunState +from contentstore.tests.utils import CourseTestCase from course_action_state.managers import CourseRerunUIStateManager -from mock import patch, Mock +from course_action_state.models import CourseRerunState +from student.auth import has_course_author_access from xmodule.contentstore.content import StaticContent from xmodule.contentstore.django import contentstore +from xmodule.modulestore import EdxJSONEncoder, ModuleStoreEnum from xmodule.modulestore.tests.factories import CourseFactory - TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/tests/test_contentstore.py b/cms/djangoapps/contentstore/tests/test_contentstore.py index 921ca7550715..5dd616ebfc4a 100644 --- a/cms/djangoapps/contentstore/tests/test_contentstore.py +++ b/cms/djangoapps/contentstore/tests/test_contentstore.py @@ -1,64 +1,55 @@ # -*- coding: utf-8 -*- import copy -import mock import shutil -import lxml.html -from lxml import etree -import ddt - from datetime import timedelta -from fs.osfs import OSFS +from functools import wraps from json import loads -from path import Path as path from textwrap import dedent -from uuid import uuid4 -from functools import wraps from unittest import SkipTest +from uuid import uuid4 +import ddt +import lxml.html +import mock from django.conf import settings from django.contrib.auth.models import User from django.test import TestCase from django.test.utils import override_settings +from edxval.api import create_video, get_videos_for_course +from fs.osfs import OSFS +from lxml import etree +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey, UsageKey +from opaque_keys.edx.locations import AssetLocation, CourseLocator +from path import Path as path -from openedx.core.lib.tempdir import mkdtemp_clean from common.test.utils import XssTestMixin -from contentstore.tests.utils import parse_json, AjaxEnabledTestClient, CourseTestCase +from contentstore.tests.utils import AjaxEnabledTestClient, CourseTestCase, get_url, parse_json +from contentstore.utils import delete_course, reverse_course_url, reverse_url from contentstore.views.component import ADVANCED_COMPONENT_TYPES - -from edxval.api import create_video, get_videos_for_course - +from course_action_state.managers import CourseActionStateItemNotFoundError +from course_action_state.models import CourseRerunState, CourseRerunUIStateManager +from django_comment_common.utils import are_permissions_roles_seeded +from openedx.core.lib.tempdir import mkdtemp_clean +from student import auth +from student.models import CourseEnrollment +from student.roles import CourseCreatorRole, CourseInstructorRole +from xmodule.capa_module import CapaDescriptor +from xmodule.contentstore.content import StaticContent from xmodule.contentstore.django import contentstore -from xmodule.contentstore.utils import restore_asset_from_trashcan, empty_asset_trashcan +from xmodule.contentstore.utils import empty_asset_trashcan, restore_asset_from_trashcan +from xmodule.course_module import CourseDescriptor, Textbook from xmodule.exceptions import InvalidVersionError from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.inheritance import own_metadata -from opaque_keys.edx.keys import UsageKey, CourseKey -from opaque_keys.edx.locations import AssetLocation, CourseLocator from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, LibraryFactory, check_mongo_calls from xmodule.modulestore.xml_exporter import export_course_to_xml from xmodule.modulestore.xml_importer import import_course_from_xml, perform_xlint - -from xmodule.capa_module import CapaDescriptor -from xmodule.course_module import CourseDescriptor, Textbook from xmodule.seq_module import SequenceDescriptor -from contentstore.utils import delete_course_and_groups, reverse_url, reverse_course_url -from django_comment_common.utils import are_permissions_roles_seeded - -from student import auth -from student.models import CourseEnrollment -from student.roles import CourseCreatorRole, CourseInstructorRole -from opaque_keys import InvalidKeyError -from contentstore.tests.utils import get_url -from course_action_state.models import CourseRerunState, CourseRerunUIStateManager - -from course_action_state.managers import CourseActionStateItemNotFoundError -from xmodule.contentstore.content import StaticContent -from xmodule.modulestore.django import modulestore - - TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex @@ -1232,7 +1223,7 @@ def test_forum_unseeding_on_delete(self): test_course_data = self.assert_created_course(number_suffix=uuid4().hex) course_id = _get_course_id(self.store, test_course_data) self.assertTrue(are_permissions_roles_seeded(course_id)) - delete_course_and_groups(course_id, self.user.id) + delete_course(course_id, self.user.id) # should raise an exception for checking permissions on deleted course with self.assertRaises(ItemNotFoundError): are_permissions_roles_seeded(course_id) @@ -1244,7 +1235,7 @@ def test_forum_unseeding_with_multiple_courses(self): # unseed the forums for the first course course_id = _get_course_id(self.store, test_course_data) - delete_course_and_groups(course_id, self.user.id) + delete_course(course_id, self.user.id) # should raise an exception for checking permissions on deleted course with self.assertRaises(ItemNotFoundError): are_permissions_roles_seeded(course_id) @@ -1264,7 +1255,7 @@ def test_course_enrollments_and_roles_on_delete(self): self.assertTrue(CourseEnrollment.is_enrolled(self.user, course_id)) self.assertTrue(self.user.roles.filter(name="Student", course_id=course_id)) - delete_course_and_groups(course_id, self.user.id) + delete_course(course_id, self.user.id) # check that user's enrollment for this course is not deleted self.assertTrue(CourseEnrollment.is_enrolled(self.user, course_id)) # check that user has form role "Student" for this course even after deleting it @@ -1286,7 +1277,7 @@ def test_course_access_groups_on_delete(self): self.assertGreater(len(instructor_role.users_with_role()), 0) # Now delete course and check that user not in instructor groups of this course - delete_course_and_groups(course_id, self.user.id) + delete_course(course_id, self.user.id) # Update our cached user since its roles have changed self.user = User.objects.get_by_natural_key(self.user.natural_key()[0]) @@ -1294,6 +1285,26 @@ def test_course_access_groups_on_delete(self): self.assertFalse(instructor_role.has_user(self.user)) self.assertEqual(len(instructor_role.users_with_role()), 0) + def test_delete_course_with_keep_instructors(self): + """ + Tests that when you delete a course with 'keep_instructors', + it does not remove any permissions of users/groups from the course + """ + test_course_data = self.assert_created_course(number_suffix=uuid4().hex) + course_id = _get_course_id(self.store, test_course_data) + + # Add and verify instructor role for the course + instructor_role = CourseInstructorRole(course_id) + instructor_role.add_users(self.user) + self.assertTrue(instructor_role.has_user(self.user)) + + delete_course(course_id, self.user.id, keep_instructors=True) + + # Update our cached user so if any change in roles can be captured + self.user = User.objects.get_by_natural_key(self.user.natural_key()[0]) + + self.assertTrue(instructor_role.has_user(self.user)) + def test_create_course_after_delete(self): """ Test that course creation works after deleting a course with the same URL @@ -1301,7 +1312,7 @@ def test_create_course_after_delete(self): test_course_data = self.assert_created_course() course_id = _get_course_id(self.store, test_course_data) - delete_course_and_groups(course_id, self.user.id) + delete_course(course_id, self.user.id) self.assert_created_course() diff --git a/cms/djangoapps/contentstore/tests/test_core_caching.py b/cms/djangoapps/contentstore/tests/test_core_caching.py index 853aff077efc..f2d08dc001ec 100644 --- a/cms/djangoapps/contentstore/tests/test_core_caching.py +++ b/cms/djangoapps/contentstore/tests/test_core_caching.py @@ -3,9 +3,9 @@ """ from django.test import TestCase - from opaque_keys.edx.locations import Location -from openedx.core.djangoapps.contentserver.caching import get_cached_content, set_cached_content, del_cached_content + +from openedx.core.djangoapps.contentserver.caching import del_cached_content, get_cached_content, set_cached_content class Content(object): diff --git a/cms/djangoapps/contentstore/tests/test_course_create_rerun.py b/cms/djangoapps/contentstore/tests/test_course_create_rerun.py index e70fc9a67142..9912416c9c24 100644 --- a/cms/djangoapps/contentstore/tests/test_course_create_rerun.py +++ b/cms/djangoapps/contentstore/tests/test_course_create_rerun.py @@ -1,26 +1,23 @@ """ Test view handler for rerun (and eventually create) """ -import ddt -from mock import patch +from datetime import datetime -from django.test.client import RequestFactory +import ddt from django.core.urlresolvers import reverse +from django.test.client import RequestFactory +from mock import patch from opaque_keys.edx.keys import CourseKey -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore +from contentstore.tests.utils import AjaxEnabledTestClient, parse_json from student.roles import CourseInstructorRole, CourseStaffRole from student.tests.factories import UserFactory -from contentstore.tests.utils import AjaxEnabledTestClient, parse_json -from datetime import datetime +from util.organizations_helpers import add_organization, get_course_organizations from xmodule.course_module import CourseFields -from util.organizations_helpers import ( - add_organization, - get_course_organizations, -) +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory @ddt.ddt diff --git a/cms/djangoapps/contentstore/tests/test_course_listing.py b/cms/djangoapps/contentstore/tests/test_course_listing.py index 0f3fa3c8ed3c..296b53bb613a 100644 --- a/cms/djangoapps/contentstore/tests/test_course_listing.py +++ b/cms/djangoapps/contentstore/tests/test_course_listing.py @@ -4,43 +4,40 @@ """ import random -from chrono import Timer -from mock import patch, Mock import ddt - -from django.conf import settings from ccx_keys.locator import CCXLocator +from chrono import Timer +from django.conf import settings from django.test import RequestFactory from django.test.client import Client +from mock import Mock, patch +from opaque_keys.edx.locations import CourseLocator from common.test.utils import XssTestMixin -from xmodule.course_module import CourseSummary - +from contentstore.tests.utils import AjaxEnabledTestClient +from contentstore.utils import delete_course from contentstore.views.course import ( - _accessible_courses_list, - _accessible_courses_list_from_groups, AccessListFallback, - get_courses_accessible_to_user, - _accessible_courses_summary_list, + _accessible_courses_iter, + _accessible_courses_list_from_groups, + _accessible_courses_summary_iter, + get_courses_accessible_to_user ) -from contentstore.utils import delete_course_and_groups -from contentstore.tests.utils import AjaxEnabledTestClient -from student.tests.factories import UserFactory +from course_action_state.models import CourseRerunState from student.roles import ( CourseInstructorRole, CourseStaffRole, GlobalStaff, - OrgStaffRole, OrgInstructorRole, - UserBasedRole, + OrgStaffRole, + UserBasedRole ) +from student.tests.factories import UserFactory +from xmodule.course_module import CourseSummary +from xmodule.error_module import ErrorDescriptor +from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory, check_mongo_calls -from xmodule.modulestore import ModuleStoreEnum -from opaque_keys.edx.locations import CourseLocator -from xmodule.error_module import ErrorDescriptor -from course_action_state.models import CourseRerunState - TOTAL_COURSES_COUNT = 10 USER_COURSES_COUNT = 1 @@ -132,11 +129,12 @@ def test_get_course_list(self): self._create_course_with_access_groups(course_location, self.user) # get courses through iterating all courses - courses_list, __ = _accessible_courses_list(self.request) + courses_iter, __ = _accessible_courses_iter(self.request) + courses_list = list(courses_iter) self.assertEqual(len(courses_list), 1) - courses_summary_list, __ = _accessible_courses_summary_list(self.request) - self.assertEqual(len(courses_summary_list), 1) + courses_summary_list, __ = _accessible_courses_summary_iter(self.request) + self.assertEqual(len(list(courses_summary_list)), 1) # get courses by reversing group name formats courses_list_by_groups, __ = _accessible_courses_list_from_groups(self.request) @@ -179,8 +177,8 @@ def test_courses_list_with_ccx_courses(self): # Verify that CCX courses are filtered out while iterating over all courses mocked_ccx_course = Mock(id=ccx_course_key) with patch('xmodule.modulestore.mixed.MixedModuleStore.get_courses', return_value=[mocked_ccx_course]): - courses_list, __ = _accessible_courses_list(self.request) - self.assertEqual(len(courses_list), 0) + courses_iter, __ = _accessible_courses_iter(self.request) + self.assertEqual(len(list(courses_iter)), 0) @ddt.data( (ModuleStoreEnum.Type.split, 'xmodule.modulestore.split_mongo.split_mongo_kvs.SplitMongoKVS'), @@ -201,8 +199,8 @@ def test_errored_course_global_staff(self, store, path_to_patch): self.assertIsInstance(self.store.get_course(course_key), ErrorDescriptor) # get courses through iterating all courses - courses_list, __ = _accessible_courses_list(self.request) - self.assertEqual(courses_list, []) + courses_iter, __ = _accessible_courses_iter(self.request) + self.assertEqual(list(courses_iter), []) # get courses by reversing group name formats courses_list_by_groups, __ = _accessible_courses_list_from_groups(self.request) @@ -231,14 +229,14 @@ def test_staff_course_listing(self, default_store, mongo_calls): # Fetch accessible courses list & verify their count courses_list_by_staff, __ = get_courses_accessible_to_user(self.request) - self.assertEqual(len(courses_list_by_staff), TOTAL_COURSES_COUNT) + self.assertEqual(len(list(courses_list_by_staff)), TOTAL_COURSES_COUNT) # Verify fetched accessible courses list is a list of CourseSummery instances self.assertTrue(all(isinstance(course, CourseSummary) for course in courses_list_by_staff)) # Now count the db queries for staff with check_mongo_calls(mongo_calls): - _accessible_courses_summary_list(self.request) + list(_accessible_courses_summary_iter(self.request)) @ddt.data( (ModuleStoreEnum.Type.split, 'xmodule.modulestore.split_mongo.split_mongo_kvs.SplitMongoKVS'), @@ -261,7 +259,8 @@ def test_errored_course_regular_access(self, store, path_to_patch): self.assertIsInstance(self.store.get_course(course_key), ErrorDescriptor) # get courses through iterating all courses - courses_list, __ = _accessible_courses_list(self.request) + courses_iter, __ = _accessible_courses_iter(self.request) + courses_list = list(courses_iter) self.assertEqual(courses_list, []) # get courses by reversing group name formats @@ -279,10 +278,12 @@ def test_get_course_list_with_invalid_course_location(self, store): self._create_course_with_access_groups(course_key, self.user, store) # get courses through iterating all courses - courses_list, __ = _accessible_courses_list(self.request) + courses_iter, __ = _accessible_courses_iter(self.request) + courses_list = list(courses_iter) self.assertEqual(len(courses_list), 1) - courses_summary_list, __ = _accessible_courses_summary_list(self.request) + courses_summary_iter, __ = _accessible_courses_summary_iter(self.request) + courses_summary_list = list(courses_summary_iter) # Verify fetched accessible courses list is a list of CourseSummery instances and only one course # is returned @@ -297,22 +298,22 @@ def test_get_course_list_with_invalid_course_location(self, store): self.assertEqual(courses_list, courses_list_by_groups) # now delete this course and re-add user to instructor group of this course - delete_course_and_groups(course_key, self.user.id) + delete_course(course_key, self.user.id) CourseInstructorRole(course_key).add_users(self.user) # Get courses through iterating all courses - courses_list, __ = _accessible_courses_list(self.request) + courses_iter, __ = _accessible_courses_iter(self.request) # Get course summaries by iterating all courses - courses_summary_list, __ = _accessible_courses_summary_list(self.request) + courses_summary_iter, __ = _accessible_courses_summary_iter(self.request) # Get courses by reversing group name formats courses_list_by_groups, __ = _accessible_courses_list_from_groups(self.request) # Test that course list returns no course self.assertEqual( - [len(courses_list), len(courses_list_by_groups), len(courses_summary_list)], + [len(list(courses_iter)), len(courses_list_by_groups), len(list(courses_summary_iter))], [0, 0, 0] ) @@ -344,13 +345,13 @@ def test_course_listing_performance(self, store, courses_list_from_group_calls, # time the get courses by iterating through all courses with Timer() as iteration_over_courses_time_1: - courses_list, __ = _accessible_courses_list(self.request) - self.assertEqual(len(courses_list), USER_COURSES_COUNT) + courses_iter, __ = _accessible_courses_iter(self.request) + self.assertEqual(len(list(courses_iter)), USER_COURSES_COUNT) # time again the get courses by iterating through all courses with Timer() as iteration_over_courses_time_2: - courses_list, __ = _accessible_courses_list(self.request) - self.assertEqual(len(courses_list), USER_COURSES_COUNT) + courses_iter, __ = _accessible_courses_iter(self.request) + self.assertEqual(len(list(courses_iter)), USER_COURSES_COUNT) # time the get courses by reversing django groups with Timer() as iteration_over_groups_time_1: @@ -362,17 +363,25 @@ def test_course_listing_performance(self, store, courses_list_from_group_calls, courses_list, __ = _accessible_courses_list_from_groups(self.request) self.assertEqual(len(courses_list), USER_COURSES_COUNT) - # test that the time taken by getting courses through reversing django groups is lower then the time - # taken by traversing through all courses (if accessible courses are relatively small) - self.assertGreaterEqual(iteration_over_courses_time_1.elapsed, iteration_over_groups_time_1.elapsed) - self.assertGreaterEqual(iteration_over_courses_time_2.elapsed, iteration_over_groups_time_2.elapsed) + # TODO (cdyer) : iteration over courses was optimized, and is now + # sometimes faster than iteration over groups. One of the following + # should be done to resolve this: + # * Iteration over groups should be sped up. + # * Iteration over groups should be removed, as it no longer saves time. + # * Or this part of the test should be removed. + + # Test that the time taken by getting courses through reversing django + # groups is lower then the time taken by traversing through all courses + # (if accessible courses are relatively small). + #self.assertGreaterEqual(iteration_over_courses_time_1.elapsed, iteration_over_groups_time_1.elapsed) + #self.assertGreaterEqual(iteration_over_courses_time_2.elapsed, iteration_over_groups_time_2.elapsed) # Now count the db queries with check_mongo_calls(courses_list_from_group_calls): _accessible_courses_list_from_groups(self.request) with check_mongo_calls(courses_list_calls): - _accessible_courses_list(self.request) + list(_accessible_courses_iter(self.request)) # Calls: # 1) query old mongo # 2) get_more on old mongo @@ -424,7 +433,7 @@ def test_course_listing_org_permissions(self, role): # Verify fetched accessible courses list is a list of CourseSummery instances and test expacted # course count is returned - self.assertEqual(len(courses_list), 2) + self.assertEqual(len(list(courses_list)), 2) self.assertTrue(all(isinstance(course, CourseSummary) for course in courses_list)) def test_course_listing_with_actions_in_progress(self): @@ -432,11 +441,17 @@ def test_course_listing_with_actions_in_progress(self): num_courses_to_create = 3 courses = [ - self._create_course_with_access_groups(CourseLocator('Org', 'CreatedCourse' + str(num), 'Run'), self.user) + self._create_course_with_access_groups( + CourseLocator('Org', 'CreatedCourse' + str(num), 'Run'), + self.user, + ) for num in range(num_courses_to_create) ] courses_in_progress = [ - self._create_course_with_access_groups(CourseLocator('Org', 'InProgressCourse' + str(num), 'Run'), self.user) + self._create_course_with_access_groups( + CourseLocator('Org', 'InProgressCourse' + str(num), 'Run'), + self.user, + ) for num in range(num_courses_to_create) ] @@ -447,7 +462,7 @@ def test_course_listing_with_actions_in_progress(self): ) # verify return values - for method in (_accessible_courses_list_from_groups, _accessible_courses_list): + for method in (_accessible_courses_list_from_groups, _accessible_courses_iter): def set_of_course_keys(course_list, key_attribute_name='id'): """Returns a python set of course keys by accessing the key with the given attribute name.""" return set(getattr(c, key_attribute_name) for c in course_list) diff --git a/cms/djangoapps/contentstore/tests/test_course_settings.py b/cms/djangoapps/contentstore/tests/test_course_settings.py index bede3bbf1506..cc6a255de79b 100644 --- a/cms/djangoapps/contentstore/tests/test_course_settings.py +++ b/cms/djangoapps/contentstore/tests/test_course_settings.py @@ -1,24 +1,25 @@ """ Tests for Studio Course Settings. """ +import copy import datetime -import ddt import json -import copy -import mock -from mock import Mock, patch import unittest +import ddt +import mock from django.conf import settings -from django.utils.timezone import UTC from django.test.utils import override_settings +from django.utils.timezone import UTC +from milestones.tests.utils import MilestonesTestCaseMixin +from mock import Mock, patch from contentstore.utils import reverse_course_url, reverse_usage_url -from models.settings.course_grading import CourseGradingModel +from models.settings.course_grading import CourseGradingModel, GRADING_POLICY_CHANGED_EVENT_TYPE, hash_grading_policy from models.settings.course_metadata import CourseMetadata from models.settings.encoder import CourseSettingsEncoder -from openedx.core.djangoapps.self_paced.models import SelfPacedConfiguration from openedx.core.djangoapps.models.course_details import CourseDetails +from openedx.core.djangoapps.self_paced.models import SelfPacedConfiguration from student.roles import CourseInstructorRole, CourseStaffRole from student.tests.factories import UserFactory from xblock_django.models import XBlockStudioConfigurationFlag @@ -27,9 +28,8 @@ from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.factories import CourseFactory from xmodule.tabs import InvalidTabsException -from milestones.tests.utils import MilestonesTestCaseMixin -from .utils import CourseTestCase, AjaxEnabledTestClient +from .utils import AjaxEnabledTestClient, CourseTestCase def get_url(course_id, handler_name='settings_handler'): @@ -86,9 +86,6 @@ class CourseDetailsViewTest(CourseTestCase, MilestonesTestCaseMixin): """ Tests for modifying content on the first course settings page (course dates, overview, etc.). """ - def setUp(self): - super(CourseDetailsViewTest, self).setUp() - def alter_field(self, url, details, field, val): """ Change the one field to the given value and then invoke the update post to see if it worked. @@ -427,18 +424,21 @@ def test_fetch_grader(self): subgrader = CourseGradingModel.fetch_grader(self.course.id, i) self.assertDictEqual(grader, subgrader, str(i) + "th graders not equal") + @mock.patch('track.event_transaction_utils.uuid4') + @mock.patch('models.settings.course_grading.tracker') + @mock.patch('contentstore.signals.signals.GRADING_POLICY_CHANGED.send') @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) - def test_update_from_json(self, store): + def test_update_from_json(self, store, send_signal, tracker, uuid): + uuid.return_value = "mockUUID" self.course = CourseFactory.create(default_store=store) - test_grader = CourseGradingModel.fetch(self.course.id) altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user) self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "Noop update") - + grading_policy_1 = self._grading_policy_hash_for_course() test_grader.graders[0]['weight'] = test_grader.graders[0].get('weight') * 2 altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user) self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "Weight[0] * 2") - + grading_policy_2 = self._grading_policy_hash_for_course() # test for bug LMS-11485 with modulestore().bulk_operations(self.course.id): new_grader = test_grader.graders[0].copy() @@ -450,49 +450,123 @@ def test_update_from_json(self, store): CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user) altered_grader = CourseGradingModel.fetch(self.course.id) self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__) - + grading_policy_3 = self._grading_policy_hash_for_course() test_grader.grade_cutoffs['D'] = 0.3 altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user) self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "cutoff add D") - + grading_policy_4 = self._grading_policy_hash_for_course() test_grader.grace_period = {'hours': 4, 'minutes': 5, 'seconds': 0} altered_grader = CourseGradingModel.update_from_json(self.course.id, test_grader.__dict__, self.user) self.assertDictEqual(test_grader.__dict__, altered_grader.__dict__, "4 hour grace period") - def test_update_grader_from_json(self): + # one for each of the calls to update_from_json() + send_signal.assert_has_calls([ + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + ]) + + # one for each of the calls to update_from_json(); the last update doesn't actually change the parts of the + # policy that get hashed + tracker.emit.assert_has_calls([ + mock.call( + GRADING_POLICY_CHANGED_EVENT_TYPE, + { + 'course_id': unicode(self.course.id), + 'event_transaction_type': 'edx.grades.grading_policy_changed', + 'grading_policy_hash': policy_hash, + 'user_id': unicode(self.user.id), + 'event_transaction_id': 'mockUUID', + } + ) for policy_hash in ( + grading_policy_1, grading_policy_2, grading_policy_3, grading_policy_4, grading_policy_4 + ) + ]) + + @mock.patch('track.event_transaction_utils.uuid4') + @mock.patch('models.settings.course_grading.tracker') + @mock.patch('contentstore.signals.signals.GRADING_POLICY_CHANGED.send') + def test_update_grader_from_json(self, send_signal, tracker, uuid): + uuid.return_value = 'mockUUID' test_grader = CourseGradingModel.fetch(self.course.id) altered_grader = CourseGradingModel.update_grader_from_json( self.course.id, test_grader.graders[1], self.user ) self.assertDictEqual(test_grader.graders[1], altered_grader, "Noop update") + grading_policy_1 = self._grading_policy_hash_for_course() test_grader.graders[1]['min_count'] = test_grader.graders[1].get('min_count') + 2 altered_grader = CourseGradingModel.update_grader_from_json( self.course.id, test_grader.graders[1], self.user) self.assertDictEqual(test_grader.graders[1], altered_grader, "min_count[1] + 2") + grading_policy_2 = self._grading_policy_hash_for_course() test_grader.graders[1]['drop_count'] = test_grader.graders[1].get('drop_count') + 1 altered_grader = CourseGradingModel.update_grader_from_json( self.course.id, test_grader.graders[1], self.user) self.assertDictEqual(test_grader.graders[1], altered_grader, "drop_count[1] + 2") + grading_policy_3 = self._grading_policy_hash_for_course() + + # one for each of the calls to update_grader_from_json() + send_signal.assert_has_calls([ + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + ]) + + # one for each of the calls to update_grader_from_json() + tracker.emit.assert_has_calls([ + mock.call( + GRADING_POLICY_CHANGED_EVENT_TYPE, + { + 'course_id': unicode(self.course.id), + 'event_transaction_type': 'edx.grades.grading_policy_changed', + 'grading_policy_hash': policy_hash, + 'user_id': unicode(self.user.id), + 'event_transaction_id': 'mockUUID', + } + ) for policy_hash in {grading_policy_1, grading_policy_2, grading_policy_3} + ]) - def test_update_cutoffs_from_json(self): + @mock.patch('track.event_transaction_utils.uuid4') + @mock.patch('models.settings.course_grading.tracker') + def test_update_cutoffs_from_json(self, tracker, uuid): + uuid.return_value = 'mockUUID' test_grader = CourseGradingModel.fetch(self.course.id) CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user) # Unlike other tests, need to actually perform a db fetch for this test since update_cutoffs_from_json # simply returns the cutoffs you send into it, rather than returning the db contents. altered_grader = CourseGradingModel.fetch(self.course.id) self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "Noop update") + grading_policy_1 = self._grading_policy_hash_for_course() test_grader.grade_cutoffs['D'] = 0.3 CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user) altered_grader = CourseGradingModel.fetch(self.course.id) self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "cutoff add D") + grading_policy_2 = self._grading_policy_hash_for_course() test_grader.grade_cutoffs['Pass'] = 0.75 CourseGradingModel.update_cutoffs_from_json(self.course.id, test_grader.grade_cutoffs, self.user) altered_grader = CourseGradingModel.fetch(self.course.id) self.assertDictEqual(test_grader.grade_cutoffs, altered_grader.grade_cutoffs, "cutoff change 'Pass'") + grading_policy_3 = self._grading_policy_hash_for_course() + + # one for each of the calls to update_cutoffs_from_json() + tracker.emit.assert_has_calls([ + mock.call( + GRADING_POLICY_CHANGED_EVENT_TYPE, + { + 'course_id': unicode(self.course.id), + 'event_transaction_type': 'edx.grades.grading_policy_changed', + 'grading_policy_hash': policy_hash, + 'user_id': unicode(self.user.id), + 'event_transaction_id': 'mockUUID', + } + ) for policy_hash in (grading_policy_1, grading_policy_2, grading_policy_3) + ]) def test_delete_grace_period(self): test_grader = CourseGradingModel.fetch(self.course.id) @@ -517,7 +591,11 @@ def test_delete_grace_period(self): # Once deleted, the grace period should simply be None self.assertEqual(None, altered_grader.grace_period, "Delete grace period") - def test_update_section_grader_type(self): + @mock.patch('track.event_transaction_utils.uuid4') + @mock.patch('models.settings.course_grading.tracker') + @mock.patch('contentstore.signals.signals.GRADING_POLICY_CHANGED.send') + def test_update_section_grader_type(self, send_signal, tracker, uuid): + uuid.return_value = 'mockUUID' # Get the descriptor and the section_grader_type and assert they are the default values descriptor = modulestore().get_item(self.course.location) section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location) @@ -530,6 +608,7 @@ def test_update_section_grader_type(self): CourseGradingModel.update_section_grader_type(self.course, 'Homework', self.user) descriptor = modulestore().get_item(self.course.location) section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location) + grading_policy_1 = self._grading_policy_hash_for_course() self.assertEqual('Homework', section_grader_type['graderType']) self.assertEqual('Homework', descriptor.format) @@ -539,34 +618,63 @@ def test_update_section_grader_type(self): CourseGradingModel.update_section_grader_type(self.course, 'notgraded', self.user) descriptor = modulestore().get_item(self.course.location) section_grader_type = CourseGradingModel.get_section_grader_type(self.course.location) + grading_policy_2 = self._grading_policy_hash_for_course() self.assertEqual('notgraded', section_grader_type['graderType']) self.assertEqual(None, descriptor.format) self.assertEqual(False, descriptor.graded) + # one for each call to update_section_grader_type() + send_signal.assert_has_calls([ + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + ]) + + tracker.emit.assert_has_calls([ + mock.call( + GRADING_POLICY_CHANGED_EVENT_TYPE, + { + 'course_id': unicode(self.course.id), + 'event_transaction_type': 'edx.grades.grading_policy_changed', + 'grading_policy_hash': policy_hash, + 'user_id': unicode(self.user.id), + 'event_transaction_id': 'mockUUID', + } + ) for policy_hash in (grading_policy_1, grading_policy_2) + ]) + + def _model_from_url(self, url_base): + response = self.client.get_json(url_base) + return json.loads(response.content) + def test_get_set_grader_types_ajax(self): """ - Test configuring the graders via ajax calls + Test creating and fetching the graders via ajax calls. """ grader_type_url_base = get_url(self.course.id, 'grading_handler') - # test get whole - response = self.client.get_json(grader_type_url_base) - whole_model = json.loads(response.content) + whole_model = self._model_from_url(grader_type_url_base) + self.assertIn('graders', whole_model) self.assertIn('grade_cutoffs', whole_model) self.assertIn('grace_period', whole_model) + # test post/update whole whole_model['grace_period'] = {'hours': 1, 'minutes': 30, 'seconds': 0} response = self.client.ajax_post(grader_type_url_base, whole_model) self.assertEqual(200, response.status_code) - response = self.client.get_json(grader_type_url_base) - whole_model = json.loads(response.content) + whole_model = self._model_from_url(grader_type_url_base) self.assertEqual(whole_model['grace_period'], {'hours': 1, 'minutes': 30, 'seconds': 0}) + # test get one grader self.assertGreater(len(whole_model['graders']), 1) # ensure test will make sense - response = self.client.get_json(grader_type_url_base + '/1') - grader_sample = json.loads(response.content) + grader_sample = self._model_from_url(grader_type_url_base + '/1') self.assertEqual(grader_sample, whole_model['graders'][1]) + + @mock.patch('contentstore.signals.signals.GRADING_POLICY_CHANGED.send') + def test_add_delete_grader(self, send_signal): + grader_type_url_base = get_url(self.course.id, 'grading_handler') + original_model = self._model_from_url(grader_type_url_base) + # test add grader new_grader = { "type": "Extra Credit", @@ -575,22 +683,31 @@ def test_get_set_grader_types_ajax(self): "short_label": None, "weight": 15, } + response = self.client.ajax_post( - '{}/{}'.format(grader_type_url_base, len(whole_model['graders'])), + '{}/{}'.format(grader_type_url_base, len(original_model['graders'])), new_grader ) + self.assertEqual(200, response.status_code) grader_sample = json.loads(response.content) - new_grader['id'] = len(whole_model['graders']) + new_grader['id'] = len(original_model['graders']) self.assertEqual(new_grader, grader_sample) - # test delete grader + + # test deleting the original grader response = self.client.delete(grader_type_url_base + '/1', HTTP_ACCEPT="application/json") + self.assertEqual(204, response.status_code) - response = self.client.get_json(grader_type_url_base) - updated_model = json.loads(response.content) + updated_model = self._model_from_url(grader_type_url_base) new_grader['id'] -= 1 # one fewer and the id mutates self.assertIn(new_grader, updated_model['graders']) - self.assertNotIn(whole_model['graders'][1], updated_model['graders']) + self.assertNotIn(original_model['graders'][1], updated_model['graders']) + send_signal.assert_has_calls([ + # once for the POST + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + # once for the DELETE + mock.call(sender=CourseGradingModel, user_id=self.user.id, course_id=self.course.id), + ]) def setup_test_set_get_section_grader_ajax(self): """ @@ -618,6 +735,9 @@ def test_set_get_section_grader_ajax(self): response = self.client.get_json(grade_type_url + '?fields=graderType') self.assertEqual(json.loads(response.content).get('graderType'), u'notgraded') + def _grading_policy_hash_for_course(self): + return hash_grading_policy(modulestore().get_course(self.course.id).grading_policy) + @ddt.ddt class CourseMetadataEditingTest(CourseTestCase): @@ -1109,13 +1229,17 @@ class CourseEnrollmentEndFieldTest(CourseTestCase): ] def setUp(self): - """ Initialize course used to test enrollment fields. """ + """ + Initialize course used to test enrollment fields. + """ super(CourseEnrollmentEndFieldTest, self).setUp() self.course = CourseFactory.create(org='edX', number='dummy', display_name='Marketing Site Course') self.course_details_url = reverse_course_url('settings_handler', unicode(self.course.id)) def _get_course_details_response(self, global_staff): - """ Return the course details page as either global or non-global staff""" + """ + Return the course details page as either global or non-global staff + """ user = UserFactory(is_staff=global_staff) CourseInstructorRole(self.course.id).add_users(user) @@ -1124,7 +1248,8 @@ def _get_course_details_response(self, global_staff): return self.client.get_html(self.course_details_url) def _verify_editable(self, response): - """ Verify that the response has expected editable fields. + """ + Verify that the response has expected editable fields. Assert that all editable field content exists and no uneditable field content exists for enrollment end fields. @@ -1137,7 +1262,8 @@ def _verify_editable(self, response): self.assertContains(response, element) def _verify_not_editable(self, response): - """ Verify that the response has expected non-editable fields. + """ + Verify that the response has expected non-editable fields. Assert that all uneditable field content exists and no editable field content exists for enrollment end fields. @@ -1151,7 +1277,8 @@ def _verify_not_editable(self, response): @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_MKTG_SITE': False}) def test_course_details_with_disabled_setting_global_staff(self): - """ Test that user enrollment end date is editable in response. + """ + Test that user enrollment end date is editable in response. Feature flag 'ENABLE_MKTG_SITE' is not enabled. User is global staff. @@ -1160,7 +1287,8 @@ def test_course_details_with_disabled_setting_global_staff(self): @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_MKTG_SITE': False}) def test_course_details_with_disabled_setting_non_global_staff(self): - """ Test that user enrollment end date is editable in response. + """ + Test that user enrollment end date is editable in response. Feature flag 'ENABLE_MKTG_SITE' is not enabled. User is non-global staff. @@ -1170,7 +1298,8 @@ def test_course_details_with_disabled_setting_non_global_staff(self): @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_MKTG_SITE': True}) @override_settings(MKTG_URLS={'ROOT': 'dummy-root'}) def test_course_details_with_enabled_setting_global_staff(self): - """ Test that user enrollment end date is editable in response. + """ + Test that user enrollment end date is editable in response. Feature flag 'ENABLE_MKTG_SITE' is enabled. User is global staff. @@ -1180,7 +1309,8 @@ def test_course_details_with_enabled_setting_global_staff(self): @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_MKTG_SITE': True}) @override_settings(MKTG_URLS={'ROOT': 'dummy-root'}) def test_course_details_with_enabled_setting_non_global_staff(self): - """ Test that user enrollment end date is not editable in response. + """ + Test that user enrollment end date is not editable in response. Feature flag 'ENABLE_MKTG_SITE' is enabled. User is non-global staff. diff --git a/cms/djangoapps/contentstore/tests/test_courseware_index.py b/cms/djangoapps/contentstore/tests/test_courseware_index.py index 155b5956d333..b6d91b3a04bc 100644 --- a/cms/djangoapps/contentstore/tests/test_courseware_index.py +++ b/cms/djangoapps/contentstore/tests/test_courseware_index.py @@ -1,19 +1,29 @@ """ Testing indexing of the courseware as it is changed """ -import ddt import json -from lazy.lazy import lazy import time from datetime import datetime -from dateutil.tz import tzutc -from mock import patch -from pytz import UTC -from uuid import uuid4 from unittest import skip +from uuid import uuid4 +import ddt +from dateutil.tz import tzutc from django.conf import settings +from lazy.lazy import lazy +from mock import patch +from pytz import UTC +from search.search_engine_base import SearchEngine +from contentstore.courseware_index import ( + CourseAboutSearchIndexer, + CoursewareSearchIndexer, + LibrarySearchIndexer, + SearchIndexingError +) +from contentstore.signals.handlers import listen_for_course_publish, listen_for_library_update +from contentstore.tests.utils import CourseTestCase +from contentstore.utils import reverse_course_url, reverse_usage_url from course_modes.models import CourseMode from openedx.core.djangoapps.models.course_details import CourseDetails from xmodule.library_tools import normalize_key_for_search @@ -25,28 +35,19 @@ from xmodule.modulestore.tests.django_utils import ( TEST_DATA_MONGO_MODULESTORE, TEST_DATA_SPLIT_MODULESTORE, - SharedModuleStoreTestCase) + SharedModuleStoreTestCase +) from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, LibraryFactory -from xmodule.modulestore.tests.mongo_connection import MONGO_PORT_NUM, MONGO_HOST +from xmodule.modulestore.tests.mongo_connection import MONGO_HOST, MONGO_PORT_NUM from xmodule.modulestore.tests.utils import ( - create_modulestore_instance, LocationMixin, - MixedSplitTestCase, MongoContentstoreBuilder + LocationMixin, + MixedSplitTestCase, + MongoContentstoreBuilder, + create_modulestore_instance ) +from xmodule.partitions.partitions import UserPartition from xmodule.tests import DATA_DIR from xmodule.x_module import XModuleMixin -from xmodule.partitions.partitions import UserPartition - -from search.search_engine_base import SearchEngine - -from contentstore.courseware_index import ( - CoursewareSearchIndexer, - LibrarySearchIndexer, - SearchIndexingError, - CourseAboutSearchIndexer, -) -from contentstore.signals import listen_for_course_publish, listen_for_library_update -from contentstore.utils import reverse_course_url, reverse_usage_url -from contentstore.tests.utils import CourseTestCase COURSE_CHILD_STRUCTURE = { "course": "chapter", @@ -131,9 +132,6 @@ class MixedWithOptionsTestCase(MixedSplitTestCase): INDEX_NAME = None DOCUMENT_TYPE = None - def setUp(self): - super(MixedWithOptionsTestCase, self).setUp() - def setup_course_base(self, store): """ base version of setup_course_base is a no-op """ pass diff --git a/cms/djangoapps/contentstore/tests/test_crud.py b/cms/djangoapps/contentstore/tests/test_crud.py index 4153646f97e3..4626c06b407d 100644 --- a/cms/djangoapps/contentstore/tests/test_crud.py +++ b/cms/djangoapps/contentstore/tests/test_crud.py @@ -1,12 +1,12 @@ from xmodule import templates -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, TEST_DATA_SPLIT_MODULESTORE -from xmodule.course_module import CourseDescriptor -from xmodule.seq_module import SequenceDescriptor from xmodule.capa_module import CapaDescriptor +from xmodule.course_module import CourseDescriptor from xmodule.html_module import HtmlDescriptor +from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.exceptions import DuplicateCourseError +from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE, ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory +from xmodule.seq_module import SequenceDescriptor class TemplateTests(ModuleStoreTestCase): diff --git a/cms/djangoapps/contentstore/tests/test_export_git.py b/cms/djangoapps/contentstore/tests/test_export_git.py index 7054b6b439f8..29d9276c4531 100644 --- a/cms/djangoapps/contentstore/tests/test_export_git.py +++ b/cms/djangoapps/contentstore/tests/test_export_git.py @@ -11,10 +11,11 @@ from django.conf import settings from django.test.utils import override_settings -from .utils import CourseTestCase import contentstore.git_export_utils as git_export_utils -from xmodule.modulestore.django import modulestore from contentstore.utils import reverse_course_url +from xmodule.modulestore.django import modulestore + +from .utils import CourseTestCase TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex diff --git a/cms/djangoapps/contentstore/tests/test_gating.py b/cms/djangoapps/contentstore/tests/test_gating.py index beacd7c2402c..b488cd31d807 100644 --- a/cms/djangoapps/contentstore/tests/test_gating.py +++ b/cms/djangoapps/contentstore/tests/test_gating.py @@ -1,18 +1,21 @@ """ Unit tests for the gating feature in Studio """ -from contentstore.signals import handle_item_deleted from milestones.tests.utils import MilestonesTestCaseMixin from mock import patch + +from contentstore.signals.handlers import handle_item_deleted from openedx.core.lib.gating import api as gating_api from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import ItemFactory, CourseFactory +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory class TestHandleItemDeleted(ModuleStoreTestCase, MilestonesTestCaseMixin): """ Test case for handle_score_changed django signal handler """ + ENABLED_SIGNALS = ['course_published'] + def setUp(self): """ Initial data setup @@ -40,16 +43,16 @@ def setUp(self): gating_api.add_prerequisite(self.course.id, self.open_seq.location) gating_api.set_required_content(self.course.id, self.gated_seq.location, self.open_seq.location, 100) - @patch('contentstore.signals.gating_api.set_required_content') - @patch('contentstore.signals.gating_api.remove_prerequisite') + @patch('contentstore.signals.handlers.gating_api.set_required_content') + @patch('contentstore.signals.handlers.gating_api.remove_prerequisite') def test_chapter_deleted(self, mock_remove_prereq, mock_set_required): """ Test gating milestone data is cleanup up when course content item is deleted """ handle_item_deleted(usage_key=self.chapter.location, user_id=0) mock_remove_prereq.assert_called_with(self.open_seq.location) mock_set_required.assert_called_with(self.open_seq.location.course_key, self.open_seq.location, None, None) - @patch('contentstore.signals.gating_api.set_required_content') - @patch('contentstore.signals.gating_api.remove_prerequisite') + @patch('contentstore.signals.handlers.gating_api.set_required_content') + @patch('contentstore.signals.handlers.gating_api.remove_prerequisite') def test_sequential_deleted(self, mock_remove_prereq, mock_set_required): """ Test gating milestone data is cleanup up when course content item is deleted """ handle_item_deleted(usage_key=self.open_seq.location, user_id=0) diff --git a/cms/djangoapps/contentstore/tests/test_i18n.py b/cms/djangoapps/contentstore/tests/test_i18n.py index 82e568cb4067..b33a4f714839 100644 --- a/cms/djangoapps/contentstore/tests/test_i18n.py +++ b/cms/djangoapps/contentstore/tests/test_i18n.py @@ -1,17 +1,19 @@ """ Tests for validate Internationalization and Module i18n service. """ -import mock import gettext from unittest import skip + +import mock from django.contrib.auth.models import User -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from contentstore.tests.utils import AjaxEnabledTestClient -from xmodule.modulestore.django import ModuleI18nService from django.utils import translation from django.utils.translation import get_language -from xmodule.modulestore.tests.factories import ItemFactory, CourseFactory + +from contentstore.tests.utils import AjaxEnabledTestClient from contentstore.views.preview import _preview_module_system +from xmodule.modulestore.django import ModuleI18nService +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory class FakeTranslations(ModuleI18nService): diff --git a/cms/djangoapps/contentstore/tests/test_import.py b/cms/djangoapps/contentstore/tests/test_import.py index f845df6e618c..005a59ddd5de 100644 --- a/cms/djangoapps/contentstore/tests/test_import.py +++ b/cms/djangoapps/contentstore/tests/test_import.py @@ -4,22 +4,23 @@ Tests for import_course_from_xml using the mongo modulestore. """ +import copy +from uuid import uuid4 + +import ddt +from django.conf import settings from django.test.client import Client from django.test.utils import override_settings -from django.conf import settings -import ddt -import copy from mock import patch from openedx.core.djangoapps.content.course_structures.tests import SignalDisconnectTestMixin -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.contentstore.django import contentstore +from xmodule.exceptions import NotFoundError from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore -from xmodule.contentstore.django import contentstore +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import check_exact_number_of_calls, check_number_of_calls from xmodule.modulestore.xml_importer import import_course_from_xml -from xmodule.exceptions import NotFoundError -from uuid import uuid4 TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex @@ -41,7 +42,7 @@ def setUp(self): self.client.login(username=self.user.username, password=self.user_password) # block_structure.update_course_in_cache cannot succeed in tests, as it needs to be run async on an lms worker - self.task_patcher = patch('openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache') + self.task_patcher = patch('openedx.core.djangoapps.content.block_structure.tasks.update_course_in_cache_v2') self._mock_lms_task = self.task_patcher.start() def tearDown(self): @@ -181,13 +182,13 @@ def test_import_performance_mongo(self): # we try to refresh the inheritance tree for each update_item in the import with check_exact_number_of_calls(store, 'refresh_cached_metadata_inheritance_tree', 28): - # _get_cached_metadata_inheritance_tree should be called twice (once for import, once on publish) - with check_exact_number_of_calls(store, '_get_cached_metadata_inheritance_tree', 2): + # _get_cached_metadata_inheritance_tree should be called once + with check_exact_number_of_calls(store, '_get_cached_metadata_inheritance_tree', 1): # with bulk-edit in progress, the inheritance tree should be recomputed only at the end of the import - # NOTE: On Jenkins, with memcache enabled, the number of calls here is only 1. - # Locally, without memcache, the number of calls is actually 2 (once more during the publish step) - with check_number_of_calls(store, '_compute_metadata_inheritance_tree', 2): + # NOTE: On Jenkins, with memcache enabled, the number of calls here is 1. + # Locally, without memcache, the number of calls is 1 (publish no longer counted) + with check_number_of_calls(store, '_compute_metadata_inheritance_tree', 1): self.load_test_import_course(create_if_not_present=False, module_store=store) @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) diff --git a/cms/djangoapps/contentstore/tests/test_import_draft_order.py b/cms/djangoapps/contentstore/tests/test_import_draft_order.py index eaf9f676f334..68d6488cf913 100644 --- a/cms/djangoapps/contentstore/tests/test_import_draft_order.py +++ b/cms/djangoapps/contentstore/tests/test_import_draft_order.py @@ -1,11 +1,11 @@ """ Tests Draft import order. """ -from xmodule.modulestore.xml_importer import import_course_from_xml +from django.conf import settings -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.django import modulestore -from django.conf import settings +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.xml_importer import import_course_from_xml TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/tests/test_import_pure_xblock.py b/cms/djangoapps/contentstore/tests/test_import_pure_xblock.py index fe4fb57f8570..05d2079fd3dc 100644 --- a/cms/djangoapps/contentstore/tests/test_import_pure_xblock.py +++ b/cms/djangoapps/contentstore/tests/test_import_pure_xblock.py @@ -2,15 +2,15 @@ Integration tests for importing courses containing pure XBlocks. """ +from django.conf import settings from xblock.core import XBlock from xblock.fields import String -from xmodule.modulestore.django import modulestore from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.xml_importer import import_course_from_xml -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.django import modulestore from xmodule.modulestore.mongo.draft import as_draft -from django.conf import settings +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.xml_importer import import_course_from_xml TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/tests/test_libraries.py b/cms/djangoapps/contentstore/tests/test_libraries.py index 6f95cd26da60..455f562f7ac8 100644 --- a/cms/djangoapps/contentstore/tests/test_libraries.py +++ b/cms/djangoapps/contentstore/tests/test_libraries.py @@ -1,30 +1,35 @@ """ Content library unit tests that require the CMS runtime. """ +import ddt from django.test.utils import override_settings +from mock import Mock, patch +from opaque_keys.edx.locator import CourseKey, LibraryLocator + from contentstore.tests.utils import AjaxEnabledTestClient, parse_json -from contentstore.utils import reverse_url, reverse_usage_url, reverse_library_url +from contentstore.utils import reverse_library_url, reverse_url, reverse_usage_url from contentstore.views.item import _duplicate_item from contentstore.views.preview import _load_preview_module from contentstore.views.tests.test_library import LIBRARY_REST_URL -import ddt -from mock import patch +from course_creators.views import add_user_with_status_granted +from openedx.core.djangoapps.content.course_structures.tests import SignalDisconnectTestMixin +from student import auth from student.auth import has_studio_read_access, has_studio_write_access from student.roles import ( - CourseInstructorRole, CourseStaffRole, CourseCreatorRole, LibraryUserRole, - OrgStaffRole, OrgInstructorRole, OrgLibraryUserRole, + CourseInstructorRole, + CourseStaffRole, + LibraryUserRole, + OrgInstructorRole, + OrgLibraryUserRole, + OrgStaffRole ) +from student.tests.factories import UserFactory +from xblock_django.user_service import DjangoXBlockUserService from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory -from mock import Mock -from opaque_keys.edx.locator import CourseKey, LibraryLocator -from openedx.core.djangoapps.content.course_structures.tests import SignalDisconnectTestMixin -from xblock_django.user_service import DjangoXBlockUserService from xmodule.x_module import STUDIO_VIEW -from student import auth -from student.tests.factories import UserFactory class LibraryTestCase(ModuleStoreTestCase): @@ -533,9 +538,13 @@ def test_creation(self): self.client.logout() self._assert_cannot_create_library(expected_code=302) # 302 redirect to login expected - # Now check that logged-in users without CourseCreator role can still create libraries + # Now check that logged-in users without CourseCreator role cannot create libraries self._login_as_non_staff_user(logout_first=False) - self.assertFalse(CourseCreatorRole().has_user(self.non_staff_user)) + with patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': True}): + self._assert_cannot_create_library(expected_code=403) # 403 user is not CourseCreator + + # Now check that logged-in users with CourseCreator role can create libraries + add_user_with_status_granted(self.user, self.non_staff_user) with patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': True}): lib_key2 = self._create_library(library="lib2", display_name="Test Library 2") library2 = modulestore().get_library(lib_key2) diff --git a/cms/djangoapps/contentstore/tests/test_orphan.py b/cms/djangoapps/contentstore/tests/test_orphan.py index 5d089bdd8f95..52c6bd80b5de 100644 --- a/cms/djangoapps/contentstore/tests/test_orphan.py +++ b/cms/djangoapps/contentstore/tests/test_orphan.py @@ -2,11 +2,12 @@ Test finding orphans via the view and django config """ import json + import ddt +from opaque_keys.edx.locator import BlockUsageLocator from contentstore.tests.utils import CourseTestCase from contentstore.utils import reverse_course_url -from opaque_keys.edx.locator import BlockUsageLocator from student.models import CourseEnrollment from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.search import path_to_location @@ -100,8 +101,8 @@ def test_get_orphans(self, default_store): self.assertIn(unicode(location), orphans) @ddt.data( - (ModuleStoreEnum.Type.split, 9, 6), - (ModuleStoreEnum.Type.mongo, 34, 13), + (ModuleStoreEnum.Type.split, 9, 5), + (ModuleStoreEnum.Type.mongo, 34, 12), ) @ddt.unpack def test_delete_orphans(self, default_store, max_mongo_calls, min_mongo_calls): diff --git a/cms/djangoapps/contentstore/tests/test_permissions.py b/cms/djangoapps/contentstore/tests/test_permissions.py index 7fb8509c3933..138034c4ffbd 100644 --- a/cms/djangoapps/contentstore/tests/test_permissions.py +++ b/cms/djangoapps/contentstore/tests/test_permissions.py @@ -5,11 +5,11 @@ from django.contrib.auth.models import User -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from contentstore.tests.utils import AjaxEnabledTestClient -from contentstore.utils import reverse_url, reverse_course_url -from student.roles import CourseInstructorRole, CourseStaffRole, OrgStaffRole, OrgInstructorRole +from contentstore.utils import reverse_course_url, reverse_url from student import auth +from student.roles import CourseInstructorRole, CourseStaffRole, OrgInstructorRole, OrgStaffRole +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase class TestCourseAccess(ModuleStoreTestCase): diff --git a/cms/djangoapps/contentstore/tests/test_proctoring.py b/cms/djangoapps/contentstore/tests/test_proctoring.py index 5be8af492834..b497874459c8 100644 --- a/cms/djangoapps/contentstore/tests/test_proctoring.py +++ b/cms/djangoapps/contentstore/tests/test_proctoring.py @@ -2,20 +2,17 @@ Tests for the edx_proctoring integration into Studio """ -from mock import patch -import ddt from datetime import datetime, timedelta + +import ddt +from edx_proctoring.api import get_all_exams_for_course, get_review_policy_by_exam_id +from mock import patch from pytz import UTC + +from contentstore.signals.handlers import listen_for_course_publish from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory -from contentstore.signals import listen_for_course_publish - -from edx_proctoring.api import ( - get_all_exams_for_course, - get_review_policy_by_exam_id -) - @ddt.ddt @patch.dict('django.conf.settings.FEATURES', {'ENABLE_SPECIAL_EXAMS': True}) diff --git a/cms/djangoapps/contentstore/tests/test_request_event.py b/cms/djangoapps/contentstore/tests/test_request_event.py index 166187ee587e..529fe4abb27d 100644 --- a/cms/djangoapps/contentstore/tests/test_request_event.py +++ b/cms/djangoapps/contentstore/tests/test_request_event.py @@ -1,8 +1,8 @@ """Tests for CMS's requests to logs""" import mock - -from django.test import TestCase from django.core.urlresolvers import reverse +from django.test import TestCase + from contentstore.views.helpers import event as cms_user_track diff --git a/cms/djangoapps/contentstore/tests/test_tasks.py b/cms/djangoapps/contentstore/tests/test_tasks.py new file mode 100644 index 000000000000..9b732b28c72b --- /dev/null +++ b/cms/djangoapps/contentstore/tests/test_tasks.py @@ -0,0 +1,163 @@ +""" +Unit tests for course import and export Celery tasks +""" +from __future__ import absolute_import, division, print_function + +import copy +import json +from uuid import uuid4 + +import mock +from django.conf import settings +from django.contrib.auth.models import User +from django.test.utils import override_settings +from opaque_keys.edx.locator import CourseLocator +from organizations.models import OrganizationCourse +from organizations.tests.factories import OrganizationFactory +from user_tasks.models import UserTaskArtifact, UserTaskStatus + +from contentstore.tasks import export_olx, rerun_course +from contentstore.tests.test_libraries import LibraryTestCase +from contentstore.tests.utils import CourseTestCase +from course_action_state.models import CourseRerunState +from openedx.core.djangoapps.embargo.models import Country, CountryAccessRule, RestrictedCourse +from xmodule.modulestore.django import modulestore + +TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) +TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex + + +def side_effect_exception(*args, **kwargs): # pylint: disable=unused-argument + """ + Side effect for mocking which raises an exception + """ + raise Exception('Boom!') + + +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +class ExportCourseTestCase(CourseTestCase): + """ + Tests of the export_olx task applied to courses + """ + + def test_success(self): + """ + Verify that a routine course export task succeeds + """ + key = str(self.course.location.course_key) + result = export_olx.delay(self.user.id, key, u'en') + status = UserTaskStatus.objects.get(task_id=result.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + artifacts = UserTaskArtifact.objects.filter(status=status) + self.assertEqual(len(artifacts), 1) + output = artifacts[0] + self.assertEqual(output.name, 'Output') + + @mock.patch('contentstore.tasks.export_course_to_xml', side_effect=side_effect_exception) + def test_exception(self, mock_export): # pylint: disable=unused-argument + """ + The export task should fail gracefully if an exception is thrown + """ + key = str(self.course.location.course_key) + result = export_olx.delay(self.user.id, key, u'en') + self._assert_failed(result, json.dumps({u'raw_error_msg': u'Boom!'})) + + def test_invalid_user_id(self): + """ + Verify that attempts to export a course as an invalid user fail + """ + user_id = User.objects.order_by(u'-id').first().pk + 100 + key = str(self.course.location.course_key) + result = export_olx.delay(user_id, key, u'en') + self._assert_failed(result, u'Unknown User ID: {}'.format(user_id)) + + def test_non_course_author(self): + """ + Verify that users who aren't authors of the course are unable to export it + """ + _, nonstaff_user = self.create_non_staff_authed_user_client() + key = str(self.course.location.course_key) + result = export_olx.delay(nonstaff_user.id, key, u'en') + self._assert_failed(result, u'Permission denied') + + def _assert_failed(self, task_result, error_message): + """ + Verify that a task failed with the specified error message + """ + status = UserTaskStatus.objects.get(task_id=task_result.id) + self.assertEqual(status.state, UserTaskStatus.FAILED) + artifacts = UserTaskArtifact.objects.filter(status=status) + self.assertEqual(len(artifacts), 1) + error = artifacts[0] + self.assertEqual(error.name, u'Error') + self.assertEqual(error.text, error_message) + + +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +class ExportLibraryTestCase(LibraryTestCase): + """ + Tests of the export_olx task applied to libraries + """ + + def test_success(self): + """ + Verify that a routine library export task succeeds + """ + key = str(self.lib_key) + result = export_olx.delay(self.user.id, key, u'en') # pylint: disable=no-member + status = UserTaskStatus.objects.get(task_id=result.id) + self.assertEqual(status.state, UserTaskStatus.SUCCEEDED) + artifacts = UserTaskArtifact.objects.filter(status=status) + self.assertEqual(len(artifacts), 1) + output = artifacts[0] + self.assertEqual(output.name, 'Output') + + +@override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) +class RerunCourseTaskTestCase(CourseTestCase): + def _rerun_course(self, old_course_key, new_course_key): + CourseRerunState.objects.initiated(old_course_key, new_course_key, self.user, 'Test Re-run') + rerun_course(str(old_course_key), str(new_course_key), self.user.id) + + def test_success(self): + """ The task should clone the OrganizationCourse and RestrictedCourse data. """ + old_course_key = self.course.id + new_course_key = CourseLocator(org=old_course_key.org, course=old_course_key.course, run='rerun') + + old_course_id = str(old_course_key) + new_course_id = str(new_course_key) + + organization = OrganizationFactory() + OrganizationCourse.objects.create(course_id=old_course_id, organization=organization) + + restricted_course = RestrictedCourse.objects.create(course_key=self.course.id) + restricted_country = Country.objects.create(country='US') + + CountryAccessRule.objects.create( + rule_type=CountryAccessRule.BLACKLIST_RULE, + restricted_course=restricted_course, + country=restricted_country + ) + + # Run the task! + self._rerun_course(old_course_key, new_course_key) + + # Verify the new course run exists + course = modulestore().get_course(new_course_key) + self.assertIsNotNone(course) + + # Verify the OrganizationCourse is cloned + self.assertEqual(OrganizationCourse.objects.count(), 2) + # This will raise an error if the OrganizationCourse object was not cloned + OrganizationCourse.objects.get(course_id=new_course_id, organization=organization) + + # Verify the RestrictedCourse and related objects are cloned + self.assertEqual(RestrictedCourse.objects.count(), 2) + restricted_course = RestrictedCourse.objects.get(course_key=new_course_key) + + self.assertEqual(CountryAccessRule.objects.count(), 2) + CountryAccessRule.objects.get( + rule_type=CountryAccessRule.BLACKLIST_RULE, + restricted_course=restricted_course, + country=restricted_country + ) diff --git a/cms/djangoapps/contentstore/tests/test_transcripts_utils.py b/cms/djangoapps/contentstore/tests/test_transcripts_utils.py index 1f2cdef39547..c7658f411ed5 100644 --- a/cms/djangoapps/contentstore/tests/test_transcripts_utils.py +++ b/cms/djangoapps/contentstore/tests/test_transcripts_utils.py @@ -1,25 +1,23 @@ # -*- coding: utf-8 -*- """ Tests for transcripts_utils. """ -import unittest -from uuid import uuid4 import copy import textwrap -from mock import patch, Mock +import unittest +from uuid import uuid4 -from django.test.utils import override_settings from django.conf import settings +from django.test.utils import override_settings from django.utils import translation -from django.utils.crypto import get_random_string - +from mock import Mock, patch from nose.plugins.skip import SkipTest -from xmodule.modulestore.tests.factories import CourseFactory +from contentstore.tests.utils import mock_requests_get from xmodule.contentstore.content import StaticContent -from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase -from xmodule.exceptions import NotFoundError from xmodule.contentstore.django import contentstore +from xmodule.exceptions import NotFoundError +from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory from xmodule.video_module import transcripts_utils -from contentstore.tests.utils import mock_requests_get TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex @@ -87,11 +85,19 @@ class TestSaveSubsToStore(SharedModuleStoreTestCase): def clear_subs_content(self): """Remove, if subtitles content exists.""" - try: - content = contentstore().find(self.content_location) - contentstore().delete(content.location) - except NotFoundError: - pass + for content_location in [self.content_location, self.content_copied_location]: + try: + content = contentstore().find(content_location) + contentstore().delete(content.location) + except NotFoundError: + pass + + @classmethod + def sub_id_to_location(cls, sub_id): + """ + A helper to compute a static file location from a subtitle id. + """ + return StaticContent.compute_location(cls.course.id, u'subs_{0}.srt.sjson'.format(sub_id)) @classmethod def setUpClass(cls): @@ -111,22 +117,31 @@ def setUpClass(cls): ] } - cls.subs_id = str(uuid4()) - filename = 'subs_{0}.srt.sjson'.format(cls.subs_id) - cls.content_location = StaticContent.compute_location(cls.course.id, filename) + # Prefix it to ensure that unicode filenames are allowed + cls.subs_id = u'uniçøde_{}'.format(uuid4()) + cls.subs_copied_id = u'cøpy_{}'.format(uuid4()) + + cls.content_location = cls.sub_id_to_location(cls.subs_id) + cls.content_copied_location = cls.sub_id_to_location(cls.subs_copied_id) # incorrect subs cls.unjsonable_subs = {1} # set can't be serialized cls.unjsonable_subs_id = str(uuid4()) - filename_unjsonable = 'subs_{0}.srt.sjson'.format(cls.unjsonable_subs_id) - cls.content_location_unjsonable = StaticContent.compute_location(cls.course.id, filename_unjsonable) + cls.content_location_unjsonable = cls.sub_id_to_location(cls.unjsonable_subs_id) def setUp(self): super(TestSaveSubsToStore, self).setUp() self.addCleanup(self.clear_subs_content) self.clear_subs_content() + def test_save_unicode_filename(self): + # Mock a video item + item = Mock(location=Mock(course_key=self.course.id)) + transcripts_utils.save_subs_to_store(self.subs, self.subs_id, self.course) + transcripts_utils.copy_or_rename_transcript(self.subs_copied_id, self.subs_id, item) + self.assertTrue(contentstore().find(self.content_copied_location)) + def test_save_subs_to_store(self): with self.assertRaises(NotFoundError): contentstore().find(self.content_location) @@ -231,17 +246,6 @@ def test_subs_for_html5_vid_with_periods(self): self.assertEqual(html5_ids[2], 'baz.1.4') self.assertEqual(html5_ids[3], 'foo') - def test_html5_id_length(self): - """ - Test that html5_id is parsed with length less than 255, as html5 ids are - used as name for transcript objects and ultimately as filename while creating - file for transcript at the time of exporting a course. - Filename can't be longer than 255 characters. - 150 chars is agreed length. - """ - html5_ids = transcripts_utils.get_html5_ids([get_random_string(255)]) - self.assertEqual(len(html5_ids[0]), 150) - @patch('xmodule.video_module.transcripts_utils.requests.get') def test_fail_downloading_subs(self, mock_get): diff --git a/cms/djangoapps/contentstore/tests/test_users_default_role.py b/cms/djangoapps/contentstore/tests/test_users_default_role.py index 326b7cf1554f..32648657ca5e 100644 --- a/cms/djangoapps/contentstore/tests/test_users_default_role.py +++ b/cms/djangoapps/contentstore/tests/test_users_default_role.py @@ -3,11 +3,10 @@ after deleting it creates same course again """ from contentstore.tests.utils import AjaxEnabledTestClient -from contentstore.utils import delete_course_and_groups, reverse_url +from contentstore.utils import delete_course, reverse_url from courseware.tests.factories import UserFactory -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase - from student.models import CourseEnrollment +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase class TestUsersDefaultRole(ModuleStoreTestCase): @@ -61,7 +60,7 @@ def test_user_forum_default_role_on_course_deletion(self): # check that user has his default "Student" forum role for this course self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key)) - delete_course_and_groups(self.course_key, self.user.id) + delete_course(self.course_key, self.user.id) # check that user's enrollment for this course is not deleted self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key)) @@ -79,7 +78,7 @@ def test_user_role_on_course_recreate(self): self.assertTrue(self.user.roles.filter(name="Student", course_id=self.course_key)) # delete this course and recreate this course with same user - delete_course_and_groups(self.course_key, self.user.id) + delete_course(self.course_key, self.user.id) resp = self._create_course_with_given_location(self.course_key) self.assertEqual(resp.status_code, 200) @@ -97,7 +96,7 @@ def test_user_role_on_course_recreate_with_change_name_case(self): # check that user has enrollment and his default "Student" forum role for this course self.assertTrue(CourseEnrollment.is_enrolled(self.user, self.course_key)) # delete this course and recreate this course with same user - delete_course_and_groups(self.course_key, self.user.id) + delete_course(self.course_key, self.user.id) # now create same course with different name case ('uppercase') new_course_key = self.course_key.replace(course=self.course_key.course.upper()) diff --git a/cms/djangoapps/contentstore/tests/test_utils.py b/cms/djangoapps/contentstore/tests/test_utils.py index c5e679ccb339..6e84ea9681d1 100644 --- a/cms/djangoapps/contentstore/tests/test_utils.py +++ b/cms/djangoapps/contentstore/tests/test_utils.py @@ -2,19 +2,18 @@ import collections from datetime import datetime, timedelta -from pytz import UTC from django.test import TestCase -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, SharedModuleStoreTestCase from opaque_keys.edx.locations import SlashSeparatedCourseKey -from xmodule.modulestore.django import modulestore -from xmodule.partitions.partitions import UserPartition, Group - -from openedx.core.djangoapps.site_configuration.tests.test_util import with_site_configuration_context +from pytz import UTC from contentstore import utils from contentstore.tests.utils import CourseTestCase +from openedx.core.djangoapps.site_configuration.tests.test_util import with_site_configuration_context +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, SharedModuleStoreTestCase +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory +from xmodule.partitions.partitions import Group, UserPartition class LMSLinksTestCase(TestCase): @@ -391,8 +390,8 @@ def test_no_visibility_set(self): def verify_all_components_visible_to_all(): # pylint: disable=invalid-name """ Verifies when group_access has not been set on anything. """ for item in (self.sequential, self.vertical, self.html, self.problem): - self.assertFalse(utils.has_children_visible_to_specific_content_groups(item)) - self.assertFalse(utils.is_visible_to_specific_content_groups(item)) + self.assertFalse(utils.has_children_visible_to_specific_partition_groups(item)) + self.assertFalse(utils.is_visible_to_specific_partition_groups(item)) verify_all_components_visible_to_all() @@ -409,16 +408,16 @@ def test_sequential_and_problem_have_group_access(self): self.set_group_access(self.vertical, {1: []}) self.set_group_access(self.problem, {2: [3, 4]}) - # Note that "has_children_visible_to_specific_content_groups" only checks immediate children. - self.assertFalse(utils.has_children_visible_to_specific_content_groups(self.sequential)) - self.assertTrue(utils.has_children_visible_to_specific_content_groups(self.vertical)) - self.assertFalse(utils.has_children_visible_to_specific_content_groups(self.html)) - self.assertFalse(utils.has_children_visible_to_specific_content_groups(self.problem)) + # Note that "has_children_visible_to_specific_partition_groups" only checks immediate children. + self.assertFalse(utils.has_children_visible_to_specific_partition_groups(self.sequential)) + self.assertTrue(utils.has_children_visible_to_specific_partition_groups(self.vertical)) + self.assertFalse(utils.has_children_visible_to_specific_partition_groups(self.html)) + self.assertFalse(utils.has_children_visible_to_specific_partition_groups(self.problem)) - self.assertTrue(utils.is_visible_to_specific_content_groups(self.sequential)) - self.assertFalse(utils.is_visible_to_specific_content_groups(self.vertical)) - self.assertFalse(utils.is_visible_to_specific_content_groups(self.html)) - self.assertTrue(utils.is_visible_to_specific_content_groups(self.problem)) + self.assertTrue(utils.is_visible_to_specific_partition_groups(self.sequential)) + self.assertFalse(utils.is_visible_to_specific_partition_groups(self.vertical)) + self.assertFalse(utils.is_visible_to_specific_partition_groups(self.html)) + self.assertTrue(utils.is_visible_to_specific_partition_groups(self.problem)) class GetUserPartitionInfoTest(ModuleStoreTestCase): @@ -493,12 +492,12 @@ def test_retrieves_partition_info_with_selected_groups(self): ] } ] - self.assertEqual(self._get_partition_info(), expected) + self.assertEqual(self._get_partition_info(schemes=["cohort", "random"]), expected) # Update group access and expect that now one group is marked as selected. self._set_group_access({0: [1]}) expected[0]["groups"][1]["selected"] = True - self.assertEqual(self._get_partition_info(), expected) + self.assertEqual(self._get_partition_info(schemes=["cohort", "random"]), expected) def test_deleted_groups(self): # Select a group that is not defined in the partition @@ -510,7 +509,7 @@ def test_deleted_groups(self): self.assertEqual(len(groups), 3) self.assertEqual(groups[2], { "id": 3, - "name": "Deleted group", + "name": "Deleted Group", "selected": True, "deleted": True }) @@ -535,9 +534,9 @@ def test_exclude_inactive_partitions(self): ), UserPartition( id=1, - name="Verification user partition", - scheme=UserPartition.get_scheme("verification"), - description="Verification user partition", + name="Completely random user partition", + scheme=UserPartition.get_scheme("random"), + description="Random user partition", groups=[ Group(id=0, name="Group C"), ], @@ -546,7 +545,7 @@ def test_exclude_inactive_partitions(self): ]) # Expect that the inactive scheme is excluded from the results - partitions = self._get_partition_info() + partitions = self._get_partition_info(schemes=["cohort", "verification"]) self.assertEqual(len(partitions), 1) self.assertEqual(partitions[0]["scheme"], "cohort") @@ -562,9 +561,9 @@ def test_exclude_partitions_with_no_groups(self): ), UserPartition( id=1, - name="Verification user partition", - scheme=UserPartition.get_scheme("verification"), - description="Verification user partition", + name="Completely random user partition", + scheme=UserPartition.get_scheme("random"), + description="Random user partition", groups=[ Group(id=0, name="Group C"), ], @@ -572,9 +571,9 @@ def test_exclude_partitions_with_no_groups(self): ]) # Expect that the partition with no groups is excluded from the results - partitions = self._get_partition_info() + partitions = self._get_partition_info(schemes=["cohort", "random"]) self.assertEqual(len(partitions), 1) - self.assertEqual(partitions[0]["scheme"], "verification") + self.assertEqual(partitions[0]["scheme"], "random") def _set_partitions(self, partitions): """Set the user partitions of the course descriptor. """ diff --git a/cms/djangoapps/contentstore/tests/tests.py b/cms/djangoapps/contentstore/tests/tests.py index f8c4d32dc0f3..528e0d4d8402 100644 --- a/cms/djangoapps/contentstore/tests/tests.py +++ b/cms/djangoapps/contentstore/tests/tests.py @@ -1,27 +1,26 @@ """ This test file will test registration, login, activation, and session activity timeouts """ +import datetime import time -import mock import unittest -from ddt import ddt, data, unpack -from django.test import TestCase -from django.test.utils import override_settings -from django.core.cache import cache +import mock +from ddt import data, ddt, unpack from django.conf import settings from django.contrib.auth.models import User +from django.core.cache import cache from django.core.urlresolvers import reverse +from django.test import TestCase +from django.test.utils import override_settings +from freezegun import freeze_time +from pytz import UTC from contentstore.models import PushNotificationConfig -from contentstore.tests.utils import parse_json, user, registration, AjaxEnabledTestClient -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from contentstore.tests.test_course_settings import CourseTestCase +from contentstore.tests.utils import AjaxEnabledTestClient, parse_json, registration, user +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory -import datetime -from pytz import UTC - -from freezegun import freeze_time class ContentStoreTestCase(ModuleStoreTestCase): @@ -302,6 +301,34 @@ def test_inactive_session_timeout(self): # re-request, and we should get a redirect to login page self.assertRedirects(resp, settings.LOGIN_REDIRECT_URL + '?next=/home/') + @mock.patch.dict(settings.FEATURES, {"ALLOW_PUBLIC_ACCOUNT_CREATION": False}) + def test_signup_button_index_page(self): + """ + Navigate to the home page and check the Sign Up button is hidden when ALLOW_PUBLIC_ACCOUNT_CREATION flag + is turned off + """ + response = self.client.get(reverse('homepage')) + self.assertNotIn('', response.content) + + @mock.patch.dict(settings.FEATURES, {"ALLOW_PUBLIC_ACCOUNT_CREATION": False}) + def test_signup_button_login_page(self): + """ + Navigate to the login page and check the Sign Up button is hidden when ALLOW_PUBLIC_ACCOUNT_CREATION flag + is turned off + """ + response = self.client.get(reverse('login')) + self.assertNotIn('', response.content) + + @mock.patch.dict(settings.FEATURES, {"ALLOW_PUBLIC_ACCOUNT_CREATION": False}) + def test_signup_link_login_page(self): + """ + Navigate to the login page and check the Sign Up link is hidden when ALLOW_PUBLIC_ACCOUNT_CREATION flag + is turned off + """ + response = self.client.get(reverse('login')) + self.assertNotIn('', + response.content) + class ForumTestCase(CourseTestCase): def setUp(self): diff --git a/cms/djangoapps/contentstore/tests/utils.py b/cms/djangoapps/contentstore/tests/utils.py index af5e7b8e526b..0c62bddbc9cc 100644 --- a/cms/djangoapps/contentstore/tests/utils.py +++ b/cms/djangoapps/contentstore/tests/utils.py @@ -3,23 +3,23 @@ ''' import json import textwrap -from mock import Mock from django.conf import settings from django.contrib.auth.models import User from django.test.client import Client -from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation +from mock import Mock +from opaque_keys.edx.locations import AssetLocation, SlashSeparatedCourseKey from contentstore.utils import reverse_url from student.models import Registration -from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore from xmodule.contentstore.django import contentstore from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.inheritance import own_metadata +from xmodule.modulestore.split_mongo.split import SplitMongoModuleStore from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import CourseFactory -from xmodule.modulestore.xml_importer import import_course_from_xml from xmodule.modulestore.tests.utils import ProceduralCourseTestMixin +from xmodule.modulestore.xml_importer import import_course_from_xml TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 8ed409fc8076..a32686736b10 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -4,25 +4,24 @@ import logging from datetime import datetime -from pytz import UTC from django.conf import settings from django.core.urlresolvers import reverse from django.utils.translation import ugettext as _ +from opaque_keys.edx.keys import CourseKey, UsageKey +from pytz import UTC + from django_comment_common.models import assign_default_role from django_comment_common.utils import seed_permissions_roles - from openedx.core.djangoapps.self_paced.models import SelfPacedConfiguration from openedx.core.djangoapps.site_configuration.models import SiteConfiguration - +from student import auth +from student.models import CourseEnrollment +from student.roles import CourseInstructorRole, CourseStaffRole from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import ItemNotFoundError -from opaque_keys.edx.keys import UsageKey, CourseKey -from student.roles import CourseInstructorRole, CourseStaffRole -from student.models import CourseEnrollment -from student import auth - +from xmodule.partitions.partitions_service import get_all_partitions_for_course log = logging.getLogger(__name__) @@ -62,22 +61,38 @@ def remove_all_instructors(course_key): instructor_role.remove_users(*instructor_role.users_with_role()) -def delete_course_and_groups(course_key, user_id): +def delete_course(course_key, user_id, keep_instructors=False): """ - This deletes the courseware associated with a course_key as well as cleaning update_item - the various user table stuff (groups, permissions, etc.) + Delete course from module store and if specified remove user and + groups permissions from course. + """ + _delete_course_from_modulestore(course_key, user_id) + + if not keep_instructors: + _remove_instructors(course_key) + + +def _delete_course_from_modulestore(course_key, user_id): + """ + Delete course from MongoDB. Deleting course will fire a signal which will result into + deletion of the courseware associated with a course_key. """ module_store = modulestore() with module_store.bulk_operations(course_key): module_store.delete_course(course_key, user_id) - print 'removing User permissions from course....' - # in the django layer, we need to remove all the user permissions groups associated with this course - try: - remove_all_instructors(course_key) - except Exception as err: - log.error("Error in deleting course groups for {0}: {1}".format(course_key, err)) + +def _remove_instructors(course_key): + """ + In the django layer, remove all the user/groups permissions associated with this course + """ + print 'removing User permissions from course....' + + try: + remove_all_instructors(course_key) + except Exception as err: + log.error("Error in deleting course groups for {0}: {1}".format(course_key, err)) def get_lms_link_for_item(location, preview=False): @@ -162,24 +177,24 @@ def is_currently_visible_to_students(xblock): return True -def has_children_visible_to_specific_content_groups(xblock): +def has_children_visible_to_specific_partition_groups(xblock): """ - Returns True if this xblock has children that are limited to specific content groups. + Returns True if this xblock has children that are limited to specific user partition groups. Note that this method is not recursive (it does not check grandchildren). """ if not xblock.has_children: return False for child in xblock.get_children(): - if is_visible_to_specific_content_groups(child): + if is_visible_to_specific_partition_groups(child): return True return False -def is_visible_to_specific_content_groups(xblock): +def is_visible_to_specific_partition_groups(xblock): """ - Returns True if this xblock has visibility limited to specific content groups. + Returns True if this xblock has visibility limited to specific user partition groups. """ if not xblock.group_access: return False @@ -283,6 +298,23 @@ def reverse_usage_url(handler_name, usage_key, kwargs=None): return reverse_url(handler_name, 'usage_key_string', usage_key, kwargs) +def get_split_group_display_name(xblock, course): + """ + Returns group name if an xblock is found in user partition groups that are suitable for the split_test module. + + Arguments: + xblock (XBlock): The courseware component. + course (XBlock): The course descriptor. + + Returns: + group name (String): Group name of the matching group xblock. + """ + for user_partition in get_user_partition_info(xblock, schemes=['random'], course=course): + for group in user_partition['groups']: + if 'Group ID {group_id}'.format(group_id=group['id']) == xblock.display_name_with_default: + return group['name'] + + def get_user_partition_info(xblock, schemes=None, course=None): """ Retrieve user partition information for an XBlock for display in editors. @@ -356,11 +388,11 @@ def get_user_partition_info(xblock, schemes=None, course=None): schemes = set(schemes) partitions = [] - for p in sorted(course.user_partitions, key=lambda p: p.name): + for p in sorted(get_all_partitions_for_course(course, active_only=True), key=lambda p: p.name): # Exclude disabled partitions, partitions with no groups defined # Also filter by scheme name if there's a filter defined. - if p.active and p.groups and (schemes is None or p.scheme.name in schemes): + if p.groups and (schemes is None or p.scheme.name in schemes): # First, add groups defined by the partition groups = [] @@ -383,7 +415,7 @@ def get_user_partition_info(xblock, schemes=None, course=None): for gid in missing_group_ids: groups.append({ "id": gid, - "name": _("Deleted group"), + "name": _("Deleted Group"), "selected": True, "deleted": True, }) @@ -391,7 +423,7 @@ def get_user_partition_info(xblock, schemes=None, course=None): # Put together the entire partition dictionary partitions.append({ "id": p.id, - "name": p.name, + "name": unicode(p.name), # Convert into a string in case ugettext_lazy was used "scheme": p.scheme.name, "groups": groups, }) @@ -411,30 +443,45 @@ def get_visibility_partition_info(xblock): Returns: dict """ - user_partitions = get_user_partition_info(xblock, schemes=["verification", "cohort"]) - cohort_partitions = [] - verification_partitions = [] - has_selected_groups = False - selected_verified_partition_id = None - - # Pre-process the partitions to make it easier to display the UI - for p in user_partitions: - has_selected = any(g["selected"] for g in p["groups"]) - has_selected_groups = has_selected_groups or has_selected - - if p["scheme"] == "cohort": - cohort_partitions.append(p) - elif p["scheme"] == "verification": - verification_partitions.append(p) - if has_selected: - selected_verified_partition_id = p["id"] + selectable_partitions = [] + # We wish to display enrollment partitions before cohort partitions. + enrollment_user_partitions = get_user_partition_info(xblock, schemes=["enrollment_track"]) + + # For enrollment partitions, we only show them if there is a selected group or + # or if the number of groups > 1. + for partition in enrollment_user_partitions: + if len(partition["groups"]) > 1 or any(group["selected"] for group in partition["groups"]): + selectable_partitions.append(partition) + + # Now add the cohort user partitions. + selectable_partitions = selectable_partitions + get_user_partition_info(xblock, schemes=["cohort"]) + + # Find the first partition with a selected group. That will be the one initially enabled in the dialog + # (if the course has only been added in Studio, only one partition should have a selected group). + selected_partition_index = -1 + + # At the same time, build up all the selected groups as they are displayed in the dialog title. + selected_groups_label = '' + + for index, partition in enumerate(selectable_partitions): + for group in partition["groups"]: + if group["selected"]: + if len(selected_groups_label) == 0: + selected_groups_label = group['name'] + else: + # Translators: This is building up a list of groups. It is marked for translation because of the + # comma, which is used as a separator between each group. + selected_groups_label = _('{previous_groups}, {current_group}').format( + previous_groups=selected_groups_label, + current_group=group['name'] + ) + if selected_partition_index == -1: + selected_partition_index = index return { - "user_partitions": user_partitions, - "cohort_partitions": cohort_partitions, - "verification_partitions": verification_partitions, - "has_selected_groups": has_selected_groups, - "selected_verified_partition_id": selected_verified_partition_id, + "selectable_partitions": selectable_partitions, + "selected_partition_index": selected_partition_index, + "selected_groups_label": selected_groups_label, } diff --git a/cms/djangoapps/contentstore/views/access.py b/cms/djangoapps/contentstore/views/access.py index a5348c702a5d..50fb74e55c4c 100644 --- a/cms/djangoapps/contentstore/views/access.py +++ b/cms/djangoapps/contentstore/views/access.py @@ -1,7 +1,7 @@ """ Helper methods for determining user access permissions in Studio """ -from student.roles import CourseInstructorRole from student import auth +from student.roles import CourseInstructorRole def get_user_role(user, course_id): diff --git a/cms/djangoapps/contentstore/views/assets.py b/cms/djangoapps/contentstore/views/assets.py index 0d00d3129d68..db72f0cdf090 100644 --- a/cms/djangoapps/contentstore/views/assets.py +++ b/cms/djangoapps/contentstore/views/assets.py @@ -1,8 +1,7 @@ +import json import logging -from functools import partial import math -import json -from pymongo import ASCENDING, DESCENDING +from functools import partial from django.conf import settings from django.contrib.auth.decorators import login_required @@ -11,11 +10,12 @@ from django.utils.translation import ugettext as _ from django.views.decorators.csrf import ensure_csrf_cookie from django.views.decorators.http import require_http_methods, require_POST +from opaque_keys.edx.keys import AssetKey, CourseKey +from pymongo import ASCENDING, DESCENDING -from edxmako.shortcuts import render_to_response from contentstore.utils import reverse_course_url from contentstore.views.exception import AssetNotFoundException -from opaque_keys.edx.keys import CourseKey, AssetKey +from edxmako.shortcuts import render_to_response from openedx.core.djangoapps.contentserver.caching import del_cached_content from student.auth import has_course_author_access from util.date_utils import get_default_time_display diff --git a/cms/djangoapps/contentstore/views/certificates.py b/cms/djangoapps/contentstore/views/certificates.py index 020f08e4c7e3..0350d4dc7e82 100644 --- a/cms/djangoapps/contentstore/views/certificates.py +++ b/cms/djangoapps/contentstore/views/certificates.py @@ -26,27 +26,26 @@ from django.conf import settings from django.contrib.auth.decorators import login_required -from django.views.decorators.csrf import ensure_csrf_cookie +from django.core.exceptions import PermissionDenied from django.http import HttpResponse from django.utils.translation import ugettext as _ +from django.views.decorators.csrf import ensure_csrf_cookie from django.views.decorators.http import require_http_methods +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import AssetKey, CourseKey -from contentstore.utils import reverse_course_url +from contentstore.utils import get_lms_link_for_certificate_web_view, reverse_course_url +from contentstore.views.assets import delete_asset +from contentstore.views.exception import AssetNotFoundException +from course_modes.models import CourseMode from edxmako.shortcuts import render_to_response -from opaque_keys.edx.keys import CourseKey, AssetKey -from opaque_keys import InvalidKeyError from eventtracking import tracker from student.auth import has_studio_write_access from student.roles import GlobalStaff -from util.db import generate_int_id, MYSQL_MAX_INT +from util.db import MYSQL_MAX_INT, generate_int_id from util.json_request import JsonResponse from xmodule.modulestore import EdxJSONEncoder from xmodule.modulestore.django import modulestore -from contentstore.views.assets import delete_asset -from contentstore.views.exception import AssetNotFoundException -from django.core.exceptions import PermissionDenied -from course_modes.models import CourseMode -from contentstore.utils import get_lms_link_for_certificate_web_view CERTIFICATE_SCHEMA_VERSION = 1 CERTIFICATE_MINIMUM_ID = 100 diff --git a/cms/djangoapps/contentstore/views/component.py b/cms/djangoapps/contentstore/views/component.py index f46a836b1042..343b6a7f84af 100644 --- a/cms/djangoapps/contentstore/views/component.py +++ b/cms/djangoapps/contentstore/views/component.py @@ -2,37 +2,31 @@ import logging -from django.http import HttpResponseBadRequest, Http404 +from django.conf import settings from django.contrib.auth.decorators import login_required -from django.views.decorators.http import require_GET from django.core.exceptions import PermissionDenied -from django.conf import settings +from django.http import Http404, HttpResponseBadRequest +from django.utils.translation import ugettext as _ +from django.views.decorators.http import require_GET from opaque_keys import InvalidKeyError from opaque_keys.edx.asides import AsideUsageKeyV1, AsideUsageKeyV2 -from xmodule.modulestore.exceptions import ItemNotFoundError -from edxmako.shortcuts import render_to_response - -from xmodule.modulestore.django import modulestore - +from opaque_keys.edx.keys import UsageKey from xblock.core import XBlock -from xblock.django.request import webob_to_django_response, django_to_webob_request +from xblock.django.request import django_to_webob_request, webob_to_django_response from xblock.exceptions import NoSuchHandlerError from xblock.plugin import PluginMissingError from xblock.runtime import Mixologist -from contentstore.utils import get_lms_link_for_item, get_xblock_aside_instance +from contentstore.utils import get_lms_link_for_item, get_xblock_aside_instance, reverse_course_url from contentstore.views.helpers import get_parent_xblock, is_unit, xblock_type_display_name -from contentstore.views.item import create_xblock_info, add_container_page_publishing_info, StudioEditModuleRuntime - -from opaque_keys.edx.keys import UsageKey - -from util.keyword_substitution import get_keywords_supported +from contentstore.views.item import StudioEditModuleRuntime, add_container_page_publishing_info, create_xblock_info +from edxmako.shortcuts import render_to_response from student.auth import has_course_author_access -from django.utils.translation import ugettext as _ - -from xblock_django.api import disabled_xblocks, authorable_xblocks +from util.keyword_substitution import get_keywords_supported +from xblock_django.api import authorable_xblocks, disabled_xblocks from xblock_django.models import XBlockStudioConfigurationFlag - +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import ItemNotFoundError __all__ = [ 'container_handler', @@ -168,6 +162,7 @@ def container_handler(request, usage_key_string): 'subsection': subsection, 'section': section, 'new_unit_category': 'vertical', + 'outline_url': '{url}?format=concise'.format(url=reverse_course_url('course_handler', course.id)), 'ancestor_xblocks': ancestor_xblocks, 'component_templates': component_templates, 'xblock_info': xblock_info, @@ -311,8 +306,9 @@ def create_support_legend_dict(): ) ) - # Add any advanced problem types. Note that these are different xblocks being stored as Advanced Problems. - if category == 'problem': + # Add any advanced problem types. Note that these are different xblocks being stored as Advanced Problems, + # currently not supported in libraries . + if category == 'problem' and not library: disabled_block_names = [block.name for block in disabled_xblocks()] advanced_problem_types = [advanced_problem_type for advanced_problem_type in ADVANCED_PROBLEM_TYPES if advanced_problem_type['component'] not in disabled_block_names] diff --git a/cms/djangoapps/contentstore/views/course.py b/cms/djangoapps/contentstore/views/course.py index 681ec7f7b218..97be034b3e87 100644 --- a/cms/djangoapps/contentstore/views/course.py +++ b/cms/djangoapps/contentstore/views/course.py @@ -5,59 +5,53 @@ import json import logging import random +from smtplib import SMTPException import string # pylint: disable=deprecated-module +import django.utils +import six +from ccx_keys.locator import CCXLocator from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied from django.core.urlresolvers import reverse -from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseNotFound, Http404 +from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseNotFound from django.shortcuts import redirect -import django.utils from django.utils.translation import ugettext as _ -from django.views.decorators.http import require_http_methods, require_GET from django.views.decorators.csrf import ensure_csrf_cookie -from smtplib import SMTPException - +from django.views.decorators.http import require_GET, require_http_methods from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locations import Location from opaque_keys.edx.locations import SlashSeparatedCourseKey +from openedx.core.djangoapps.content.course_overviews.models import CourseOverview +from openedx.core.djangoapps.waffle_utils import WaffleSwitchNamespace -from .component import ( - ADVANCED_COMPONENT_TYPES, -) -from .item import create_xblock_info -from .library import LIBRARIES_ENABLED -from ccx_keys.locator import CCXLocator from contentstore.course_group_config import ( COHORT_SCHEME, - GroupConfiguration, - GroupConfigurationsValidationError, + ENROLLMENT_SCHEME, RANDOM_SCHEME, + GroupConfiguration, + GroupConfigurationsValidationError ) -from contentstore.course_info_model import get_course_updates, update_course_updates, delete_course_update +from contentstore.course_info_model import delete_course_update, get_course_updates, update_course_updates from contentstore.courseware_index import CoursewareSearchIndexer, SearchIndexingError from contentstore.push_notification import push_notification_enabled from contentstore.tasks import rerun_course from contentstore.utils import ( add_instructor, - initialize_permissions, get_lms_link_for_item, + initialize_permissions, remove_all_instructors, reverse_course_url, reverse_library_url, - reverse_usage_url, reverse_url, + reverse_usage_url ) -from contentstore.views.entrance_exam import ( - create_entrance_exam, - delete_entrance_exam, - update_entrance_exam, -) +from contentstore.views.entrance_exam import create_entrance_exam, delete_entrance_exam, update_entrance_exam from course_action_state.managers import CourseActionStateItemNotFoundError from course_action_state.models import CourseRerunState, CourseRerunUIStateManager -from course_creators.views import get_course_creator_status, add_user_with_status_unrequested +from course_creators.views import add_user_with_status_unrequested, get_course_creator_status from edxmako.shortcuts import render_to_response from edxmako.shortcuts import render_to_string from microsite_configuration import microsite @@ -65,22 +59,18 @@ from models.settings.course_metadata import CourseMetadata from models.settings.encoder import CourseSettingsEncoder from openedx.core.djangoapps.content.course_structures.api.v0 import api, errors -from openedx.core.djangoapps.credit.api import is_credit_course, get_credit_requirements +from openedx.core.djangoapps.credit.api import get_credit_requirements, is_credit_course from openedx.core.djangoapps.credit.tasks import update_credit_course_requirements from openedx.core.djangoapps.models.course_details import CourseDetails -from openedx.core.djangoapps.programs.models import ProgramsApiConfig -from openedx.core.djangoapps.programs.utils import get_programs from openedx.core.djangoapps.self_paced.models import SelfPacedConfiguration from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers +from openedx.core.djangolib.js_utils import dump_js_escaped_json from openedx.core.lib.course_tabs import CourseTabPluginManager from openedx.core.lib.courses import course_image_url -from openedx.core.djangolib.js_utils import dump_js_escaped_json from student import auth -from student.auth import has_course_author_access, has_studio_write_access, has_studio_read_access -from student.roles import ( - CourseInstructorRole, CourseStaffRole, CourseCreatorRole, GlobalStaff, UserBasedRole -) -from util.course import get_lms_link_for_about_page +from student.auth import has_course_author_access, has_studio_read_access, has_studio_write_access +from student.roles import CourseCreatorRole, CourseInstructorRole, CourseStaffRole, GlobalStaff, UserBasedRole +from util.course import get_link_for_about_page from util.date_utils import get_default_time_display from util.json_request import JsonResponse, JsonResponseBadRequest, expect_json from util.keyword_substitution import get_keywords_supported, substitute_keywords_with_data @@ -88,24 +78,22 @@ is_entrance_exams_enabled, is_prerequisite_courses_enabled, is_valid_course_key, - set_prerequisite_courses, -) -from util.organizations_helpers import ( - add_organization_course, - get_organization_by_short_name, - organizations_enabled, + set_prerequisite_courses ) +from util.organizations_helpers import add_organization_course, get_organization_by_short_name, organizations_enabled from util.string_utils import _has_non_ascii_characters from xblock_django.api import deprecated_xblocks from xmodule.contentstore.content import StaticContent -from xmodule.course_module import CourseFields -from xmodule.course_module import DEFAULT_START_DATE +from xmodule.course_module import DEFAULT_START_DATE, CourseFields from xmodule.error_module import ErrorDescriptor from xmodule.modulestore import EdxJSONEncoder from xmodule.modulestore.django import modulestore -from xmodule.modulestore.exceptions import ItemNotFoundError, DuplicateCourseError +from xmodule.modulestore.exceptions import DuplicateCourseError, ItemNotFoundError from xmodule.tabs import CourseTab, CourseTabList, InvalidTabsException +from .component import ADVANCED_COMPONENT_TYPES +from .item import create_xblock_info +from .library import LIBRARIES_ENABLED, get_library_creator_status log = logging.getLogger(__name__) @@ -121,6 +109,8 @@ 'group_configurations_list_handler', 'group_configurations_detail_handler', 'send_test_enrollment_email'] +WAFFLE_NAMESPACE = 'studio_home' + class AccessListFallback(Exception): """ @@ -345,11 +335,16 @@ def _course_outline_json(request, course_module): """ Returns a JSON representation of the course module and recursively all of its children. """ + is_concise = request.GET.get('format') == 'concise' + include_children_predicate = lambda xblock: not xblock.category == 'vertical' + if is_concise: + include_children_predicate = lambda xblock: xblock.has_children return create_xblock_info( course_module, include_child_info=True, - course_outline=True, - include_children_predicate=lambda xblock: not xblock.category == 'vertical', + course_outline=False if is_concise else True, + include_children_predicate=include_children_predicate, + is_concise=is_concise, user=request.user ) @@ -361,15 +356,22 @@ def get_in_process_course_actions(request): return [ course for course in CourseRerunState.objects.find_all( - exclude_args={'state': CourseRerunUIStateManager.State.SUCCEEDED}, should_display=True + exclude_args={'state': CourseRerunUIStateManager.State.SUCCEEDED}, + should_display=True, ) if has_studio_read_access(request.user, course.course_key) ] -def _accessible_courses_summary_list(request): +def _accessible_courses_summary_iter(request, org=None): """ List all courses available to the logged in user by iterating through all the courses + + Arguments: + request: the request object + org (string): if not None, this value will limit the courses returned. An empty + string will result in no courses, and otherwise only courses with the + specified org will be returned. The default value is None. """ def course_filter(course_summary): """ @@ -381,15 +383,19 @@ def course_filter(course_summary): return False return has_studio_read_access(request.user, course_summary.id) - - courses_summary = filter(course_filter, modulestore().get_course_summaries()) + if org is not None: + courses_summary = [] if org == '' else CourseOverview.get_all_courses(orgs=[org]) + else: + courses_summary = modulestore().get_course_summaries() + courses_summary = six.moves.filter(course_filter, courses_summary) in_process_course_actions = get_in_process_course_actions(request) return courses_summary, in_process_course_actions -def _accessible_courses_list(request): +def _accessible_courses_iter(request): """ - List all courses available to the logged in user by iterating through all the courses + List all courses available to the logged in user by iterating through all the courses. + This method is only used by tests. """ def course_filter(course): """ @@ -410,7 +416,7 @@ def course_filter(course): return has_studio_read_access(request.user, course.id) - courses = filter(course_filter, modulestore().get_courses()) + courses = six.moves.filter(course_filter, modulestore().get_courses()) in_process_course_actions = get_in_process_course_actions(request) return courses, in_process_course_actions @@ -459,12 +465,12 @@ def filter_ccx(course_access): return courses_list.values(), in_process_course_actions -def _accessible_libraries_list(user): +def _accessible_libraries_iter(user): """ List all libraries available to the logged in user by iterating through all libraries """ # No need to worry about ErrorDescriptors - split's get_libraries() never returns them. - return [lib for lib in modulestore().get_libraries() if has_studio_read_access(user, lib.location.library_key)] + return (lib for lib in modulestore().get_libraries() if has_studio_read_access(user, lib.location.library_key)) @login_required @@ -473,38 +479,38 @@ def course_listing(request): """ List all courses available to the logged in user """ - courses, in_process_course_actions = get_courses_accessible_to_user(request) - if not settings.SPLIT_STUDIO_HOME and LIBRARIES_ENABLED: - libraries = _accessible_libraries_list(request.user) - else: - libraries = [] - - programs_config = ProgramsApiConfig.current() - raw_programs = get_programs(request.user) if programs_config.is_studio_tab_enabled else [] + optimization_enabled = GlobalStaff().has_user(request.user) and \ + WaffleSwitchNamespace(name=WAFFLE_NAMESPACE).is_enabled(u'enable_global_staff_optimization') - # Sort programs alphabetically by name. - # TODO: Support ordering in the Programs API itself. - programs = sorted(raw_programs, key=lambda p: p['name'].lower()) + if optimization_enabled: + org = request.GET.get('org', '') + show_libraries = LIBRARIES_ENABLED and request.GET.get('libraries', 'false').lower() == 'true' + else: + org = None + show_libraries = LIBRARIES_ENABLED + courses_iter, in_process_course_actions = get_courses_accessible_to_user(request, org) + user = request.user + libraries = _accessible_libraries_iter(request.user) if show_libraries else [] def format_in_process_course_view(uca): """ Return a dict of the data which the view requires for each unsucceeded course """ return { - 'display_name': uca.display_name, - 'course_key': unicode(uca.course_key), - 'org': uca.course_key.org, - 'number': uca.course_key.course, - 'run': uca.course_key.run, - 'is_failed': True if uca.state == CourseRerunUIStateManager.State.FAILED else False, - 'is_in_progress': True if uca.state == CourseRerunUIStateManager.State.IN_PROGRESS else False, - 'dismiss_link': reverse_course_url( - 'course_notifications_handler', + u'display_name': uca.display_name, + u'course_key': unicode(uca.course_key), + u'org': uca.course_key.org, + u'number': uca.course_key.course, + u'run': uca.course_key.run, + u'is_failed': True if uca.state == CourseRerunUIStateManager.State.FAILED else False, + u'is_in_progress': True if uca.state == CourseRerunUIStateManager.State.IN_PROGRESS else False, + u'dismiss_link': reverse_course_url( + u'course_notifications_handler', uca.course_key, kwargs={ - 'action_state_id': uca.id, + u'action_state_id': uca.id, }, - ) if uca.state == CourseRerunUIStateManager.State.FAILED else '' + ) if uca.state == CourseRerunUIStateManager.State.FAILED else u'' } def format_library_for_view(library): @@ -512,33 +518,30 @@ def format_library_for_view(library): Return a dict of the data which the view requires for each library """ return { - 'display_name': library.display_name, - 'library_key': unicode(library.location.library_key), - 'url': reverse_library_url('library_handler', unicode(library.location.library_key)), - 'org': library.display_org_with_default, - 'number': library.display_number_with_default, - 'can_edit': has_studio_write_access(request.user, library.location.library_key), + u'display_name': library.display_name, + u'library_key': unicode(library.location.library_key), + u'url': reverse_library_url(u'library_handler', unicode(library.location.library_key)), + u'org': library.display_org_with_default, + u'number': library.display_number_with_default, + u'can_edit': has_studio_write_access(request.user, library.location.library_key), } - courses = _remove_in_process_courses(courses, in_process_course_actions) + courses_iter = _remove_in_process_courses(courses_iter, in_process_course_actions) in_process_course_actions = [format_in_process_course_view(uca) for uca in in_process_course_actions] - return render_to_response('index.html', { - 'courses': courses, - 'split_studio_home': settings.SPLIT_STUDIO_HOME, - 'in_process_course_actions': in_process_course_actions, - 'libraries_enabled': LIBRARIES_ENABLED, - 'libraries': [format_library_for_view(lib) for lib in libraries], - 'show_new_library_button': LIBRARIES_ENABLED and request.user.is_active, - 'user': request.user, - 'request_course_creator_url': reverse('contentstore.views.request_course_creator'), - 'course_creator_status': _get_course_creator_status(request.user), - 'rerun_creator_status': GlobalStaff().has_user(request.user), - 'allow_unicode_course_id': settings.FEATURES.get('ALLOW_UNICODE_COURSE_ID', False), - 'allow_course_reruns': settings.FEATURES.get('ALLOW_COURSE_RERUNS', True), - 'is_programs_enabled': programs_config.is_studio_tab_enabled and request.user.is_staff, - 'programs': programs, - 'program_authoring_url': reverse('programs'), + return render_to_response(u'index.html', { + u'courses': list(courses_iter), + u'in_process_course_actions': in_process_course_actions, + u'libraries_enabled': show_libraries, + u'libraries': [format_library_for_view(lib) for lib in libraries], + u'show_new_library_button': show_libraries and get_library_creator_status(user), + u'user': user, + u'request_course_creator_url': reverse(u'contentstore.views.request_course_creator'), + u'course_creator_status': _get_course_creator_status(user), + u'rerun_creator_status': GlobalStaff().has_user(user), + u'allow_unicode_course_id': settings.FEATURES.get(u'ALLOW_UNICODE_COURSE_ID', False), + u'allow_course_reruns': settings.FEATURES.get(u'ALLOW_COURSE_RERUNS', True), + u'optimization_enabled': optimization_enabled }) @@ -599,16 +602,14 @@ def _deprecated_blocks_info(course_module, deprecated_block_types): Returns: Dict with following keys: - block_types (list): list containing types of all deprecated blocks - block_types_enabled (bool): True if any or all `deprecated_blocks` present in Advanced Module List else False - blocks (list): List of `deprecated_block_types` component names and their parent's url + deprecated_enabled_block_types (list): list containing all deprecated blocks types enabled on this course + blocks (list): List of `deprecated_enabled_block_types` instances and their parent's url advance_settings_url (str): URL to advance settings page """ data = { - 'block_types': deprecated_block_types, - 'block_types_enabled': any( - block_type in course_module.advanced_modules for block_type in deprecated_block_types - ), + 'deprecated_enabled_block_types': [ + block_type for block_type in course_module.advanced_modules if block_type in deprecated_block_types + ], 'blocks': [], 'advance_settings_url': reverse_course_url('advanced_settings_handler', course_module.id) } @@ -677,25 +678,33 @@ def course_index(request, course_key): }) -def get_courses_accessible_to_user(request): +def get_courses_accessible_to_user(request, org=None): """ Try to get all courses by first reversing django groups and fallback to old method if it fails Note: overhead of pymongo reads will increase if getting courses from django groups fails + + Arguments: + request: the request object + org (string): for global staff users ONLY, this value will be used to limit + the courses returned. A value of None will have no effect (all courses + returned), an empty string will result in no courses, and otherwise only courses with the + specified org will be returned. The default value is None. + """ if GlobalStaff().has_user(request.user): # user has global access so no need to get courses from django groups - courses, in_process_course_actions = _accessible_courses_summary_list(request) + courses, in_process_course_actions = _accessible_courses_summary_iter(request, org) else: try: courses, in_process_course_actions = _accessible_courses_list_from_groups(request) except AccessListFallback: # user have some old groups or there was some error getting courses from django groups # so fallback to iterating through all courses - courses, in_process_course_actions = _accessible_courses_summary_list(request) + courses, in_process_course_actions = _accessible_courses_summary_iter(request) return courses, in_process_course_actions -def _remove_in_process_courses(courses, in_process_course_actions): +def _remove_in_process_courses(courses_iter, in_process_course_actions): """ removes any in-process courses in courses list. in-process actually refers to courses that are in the process of being generated for re-run @@ -721,13 +730,12 @@ def format_course_for_view(course): 'run': course.location.run } - in_process_action_course_keys = [uca.course_key for uca in in_process_course_actions] - courses = [ + in_process_action_course_keys = {uca.course_key for uca in in_process_course_actions} + return ( format_course_for_view(course) - for course in courses + for course in courses_iter if not isinstance(course, ErrorDescriptor) and (course.id not in in_process_action_course_keys) - ] - return courses + ) def course_outline_initial_state(locator_to_show, course_structure): @@ -1092,7 +1100,7 @@ def settings_handler(request, course_key_string): settings_context = { 'context_course': course_module, 'course_locator': course_key, - 'lms_link_for_about_page': get_lms_link_for_about_page(course_key), + 'lms_link_for_about_page': get_link_for_about_page(course_module), 'course_image_url': course_image_url(course_module, 'course_image'), 'banner_image_url': course_image_url(course_module, 'banner_image'), 'video_thumbnail_image_url': course_image_url(course_module, 'video_thumbnail_image'), @@ -1118,10 +1126,10 @@ def settings_handler(request, course_key_string): if is_prerequisite_courses_enabled(): courses, in_process_course_actions = get_courses_accessible_to_user(request) # exclude current course from the list of available courses - courses = [course for course in courses if course.id != course_key] + courses = (course for course in courses if course.id != course_key) if courses: courses = _remove_in_process_courses(courses, in_process_course_actions) - settings_context.update({'possible_pre_requisite_courses': courses}) + settings_context.update({'possible_pre_requisite_courses': list(courses)}) if credit_eligibility_enabled: if is_credit_course(course_key): @@ -1577,7 +1585,7 @@ def remove_content_or_experiment_group(request, store, course, configuration, gr return JsonResponse(status=404) group_id = int(group_id) - usages = GroupConfiguration.get_content_groups_usage_info(store, course) + usages = GroupConfiguration.get_partitions_usage_info(store, course) used = group_id in usages if used: @@ -1625,7 +1633,24 @@ def group_configurations_list_handler(request, course_key_string): else: experiment_group_configurations = None - content_group_configuration = GroupConfiguration.get_or_create_content_group(store, course) + all_partitions = GroupConfiguration.get_all_user_partition_details(store, course) + should_show_enrollment_track = False + group_schemes = [] + for partition in all_partitions: + group_schemes.append(partition['scheme']) + if partition['scheme'] == ENROLLMENT_SCHEME: + enrollment_track_configuration = partition + should_show_enrollment_track = len(enrollment_track_configuration['groups']) > 1 + + # Remove the enrollment track partition and add it to the front of the list if it should be shown. + all_partitions.remove(partition) + if should_show_enrollment_track: + all_partitions.insert(0, partition) + + # Add empty content group if there is no COHORT User Partition in the list. + # This will add ability to add new groups in the view. + if COHORT_SCHEME not in group_schemes: + all_partitions.append(GroupConfiguration.get_or_create_content_group(store, course)) return render_to_response('group_configurations.html', { 'context_course': course, @@ -1633,7 +1658,8 @@ def group_configurations_list_handler(request, course_key_string): 'course_outline_url': course_outline_url, 'experiment_group_configurations': experiment_group_configurations, 'should_show_experiment_groups': should_show_experiment_groups, - 'content_group_configuration': content_group_configuration + 'all_group_configurations': all_partitions, + 'should_show_enrollment_track': should_show_enrollment_track }) elif "application/json" in request.META.get('HTTP_ACCEPT'): if request.method == 'POST': @@ -1727,6 +1753,7 @@ def _get_course_creator_status(user): If the user passed in has not previously visited the index page, it will be added with status 'unrequested' if the course creator group is in use. """ + if user.is_staff: course_creator_status = 'granted' elif settings.FEATURES.get('DISABLE_COURSE_CREATION', False): diff --git a/cms/djangoapps/contentstore/views/entrance_exam.py b/cms/djangoapps/contentstore/views/entrance_exam.py index 9dba4ae26114..493c2126bc42 100644 --- a/cms/djangoapps/contentstore/views/entrance_exam.py +++ b/cms/djangoapps/contentstore/views/entrance_exam.py @@ -2,25 +2,25 @@ Entrance Exams view module -- handles all requests related to entrance exam management via Studio Intended to be utilized as an AJAX callback handler, versus a proper view/screen """ -from functools import wraps import logging +from functools import wraps +from django.conf import settings from django.contrib.auth.decorators import login_required -from django.views.decorators.csrf import ensure_csrf_cookie from django.http import HttpResponse, HttpResponseBadRequest +from django.utils.translation import ugettext as _ +from django.views.decorators.csrf import ensure_csrf_cookie +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey, UsageKey -from openedx.core.djangolib.js_utils import dump_js_escaped_json from contentstore.views.helpers import create_xblock, remove_entrance_exam_graders from contentstore.views.item import delete_item from models.settings.course_metadata import CourseMetadata -from opaque_keys.edx.keys import CourseKey, UsageKey -from opaque_keys import InvalidKeyError +from openedx.core.djangolib.js_utils import dump_js_escaped_json from student.auth import has_course_author_access from util import milestones_helpers from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import ItemNotFoundError -from django.conf import settings -from django.utils.translation import ugettext as _ __all__ = ['entrance_exam', ] diff --git a/cms/djangoapps/contentstore/views/error.py b/cms/djangoapps/contentstore/views/error.py index 6f58ea296df9..e1f1d6b21efa 100644 --- a/cms/djangoapps/contentstore/views/error.py +++ b/cms/djangoapps/contentstore/views/error.py @@ -1,9 +1,10 @@ # pylint: disable=missing-docstring,unused-argument -from django.http import (HttpResponse, HttpResponseServerError, - HttpResponseNotFound) -from edxmako.shortcuts import render_to_string, render_to_response import functools + +from django.http import HttpResponse, HttpResponseNotFound, HttpResponseServerError + +from edxmako.shortcuts import render_to_response, render_to_string from openedx.core.djangolib.js_utils import dump_js_escaped_json __all__ = ['not_found', 'server_error', 'render_404', 'render_500'] diff --git a/cms/djangoapps/contentstore/views/export_git.py b/cms/djangoapps/contentstore/views/export_git.py index bb1f1d4c0e04..f56264059183 100644 --- a/cms/djangoapps/contentstore/views/export_git.py +++ b/cms/djangoapps/contentstore/views/export_git.py @@ -7,14 +7,14 @@ from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied -from django.views.decorators.csrf import ensure_csrf_cookie from django.utils.translation import ugettext as _ +from django.views.decorators.csrf import ensure_csrf_cookie +from opaque_keys.edx.keys import CourseKey -from student.auth import has_course_author_access import contentstore.git_export_utils as git_export_utils from edxmako.shortcuts import render_to_response +from student.auth import has_course_author_access from xmodule.modulestore.django import modulestore -from opaque_keys.edx.keys import CourseKey log = logging.getLogger(__name__) diff --git a/cms/djangoapps/contentstore/views/helpers.py b/cms/djangoapps/contentstore/views/helpers.py index 29e571c5b690..29e0f0bc6b6f 100644 --- a/cms/djangoapps/contentstore/views/helpers.py +++ b/cms/djangoapps/contentstore/views/helpers.py @@ -4,24 +4,23 @@ from __future__ import absolute_import -from uuid import uuid4 import urllib +from uuid import uuid4 from django.conf import settings from django.http import HttpResponse from django.utils.translation import ugettext as _ - -from edxmako.shortcuts import render_to_string from opaque_keys.edx.keys import UsageKey from xblock.core import XBlock -import dogstats_wrapper as dog_stats_api -from xmodule.modulestore.django import modulestore -from xmodule.x_module import DEPRECATION_VSCOMPAT_EVENT -from xmodule.tabs import StaticTab +import dogstats_wrapper as dog_stats_api from contentstore.utils import reverse_course_url, reverse_library_url, reverse_usage_url +from edxmako.shortcuts import render_to_string from models.settings.course_grading import CourseGradingModel from util.milestones_helpers import is_entrance_exams_enabled +from xmodule.modulestore.django import modulestore +from xmodule.tabs import StaticTab +from xmodule.x_module import DEPRECATION_VSCOMPAT_EVENT __all__ = ['event'] diff --git a/cms/djangoapps/contentstore/views/import_export.py b/cms/djangoapps/contentstore/views/import_export.py index 610279baff5b..f06ef7c57ec4 100644 --- a/cms/djangoapps/contentstore/views/import_export.py +++ b/cms/djangoapps/contentstore/views/import_export.py @@ -3,52 +3,42 @@ courses """ import base64 +import json import logging import os import re import shutil -import tarfile -from path import Path as path -from tempfile import mkdtemp from django.conf import settings from django.contrib.auth.decorators import login_required -from django.core.exceptions import SuspiciousOperation, PermissionDenied -from django.core.files.temp import NamedTemporaryFile +from django.core.exceptions import PermissionDenied +from django.core.files import File from django.core.servers.basehttp import FileWrapper -from django.http import HttpResponse, HttpResponseNotFound, Http404 +from django.db import transaction +from django.http import Http404, HttpResponse, HttpResponseNotFound from django.utils.translation import ugettext as _ from django.views.decorators.csrf import ensure_csrf_cookie -from django.views.decorators.http import require_http_methods, require_GET - -import dogstats_wrapper as dog_stats_api -from edxmako.shortcuts import render_to_response -from xmodule.contentstore.django import contentstore -from xmodule.exceptions import SerializationError -from xmodule.modulestore.django import modulestore +from django.views.decorators.http import require_GET, require_http_methods from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import LibraryLocator -from xmodule.modulestore.xml_importer import import_course_from_xml, import_library_from_xml -from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml -from xmodule.modulestore import COURSE_ROOT, LIBRARY_ROOT +from path import Path as path +from six import text_type +from user_tasks.conf import settings as user_tasks_settings +from user_tasks.models import UserTaskArtifact, UserTaskStatus +from contentstore.storage import course_import_export_storage +from contentstore.tasks import CourseExportTask, CourseImportTask, create_export_tarball, export_olx, import_olx +from contentstore.utils import reverse_course_url, reverse_library_url +from edxmako.shortcuts import render_to_response from student.auth import has_course_author_access - -from openedx.core.lib.extract_tar import safetar_extractall from util.json_request import JsonResponse from util.views import ensure_valid_course_key -from models.settings.course_metadata import CourseMetadata -from contentstore.views.entrance_exam import ( - add_entrance_exam_milestone, - remove_entrance_exam_milestone_reference -) - -from contentstore.utils import reverse_course_url, reverse_usage_url, reverse_library_url - +from xmodule.exceptions import SerializationError +from xmodule.modulestore.django import modulestore __all__ = [ 'import_handler', 'import_status_handler', - 'export_handler', + 'export_handler', 'export_output_handler', 'export_status_handler', ] @@ -58,7 +48,10 @@ # Regex to capture Content-Range header ranges. CONTENT_RE = re.compile(r"(?P\d{1,11})-(?P\d{1,11})/(?P\d{1,11})") +STATUS_FILTERS = user_tasks_settings.USER_TASKS_STATUS_FILTERS + +@transaction.non_atomic_requests @login_required @ensure_csrf_cookie @require_http_methods(("GET", "POST", "PUT")) @@ -76,26 +69,13 @@ def import_handler(request, course_key_string): courselike_key = CourseKey.from_string(course_key_string) library = isinstance(courselike_key, LibraryLocator) if library: - root_name = LIBRARY_ROOT successful_url = reverse_library_url('library_handler', courselike_key) context_name = 'context_library' courselike_module = modulestore().get_library(courselike_key) - import_func = import_library_from_xml else: - root_name = COURSE_ROOT successful_url = reverse_course_url('course_handler', courselike_key) context_name = 'context_course' courselike_module = modulestore().get_course(courselike_key) - import_func = import_course_from_xml - return _import_handler( - request, courselike_key, root_name, successful_url, context_name, courselike_module, import_func - ) - - -def _import_handler(request, courselike_key, root_name, successful_url, context_name, courselike_module, import_func): - """ - Parameterized function containing the meat of import_handler. - """ if not has_course_author_access(request.user, courselike_key): raise PermissionDenied() @@ -103,235 +83,7 @@ def _import_handler(request, courselike_key, root_name, successful_url, context_ if request.method == 'GET': raise NotImplementedError('coming soon') else: - # Do everything in a try-except block to make sure everything is properly cleaned up. - try: - data_root = path(settings.GITHUB_REPO_ROOT) - subdir = base64.urlsafe_b64encode(repr(courselike_key)) - course_dir = data_root / subdir - filename = request.FILES['course-data'].name - - # Use sessions to keep info about import progress - session_status = request.session.setdefault("import_status", {}) - courselike_string = unicode(courselike_key) + filename - _save_request_status(request, courselike_string, 0) - - # If the course has an entrance exam then remove it and its corresponding milestone. - # current course state before import. - if root_name == COURSE_ROOT: - if courselike_module.entrance_exam_enabled: - remove_entrance_exam_milestone_reference(request, courselike_key) - log.info( - "entrance exam milestone content reference for course %s has been removed", - courselike_module.id - ) - - if not filename.endswith('.tar.gz'): - _save_request_status(request, courselike_string, -1) - return JsonResponse( - { - 'ErrMsg': _('We only support uploading a .tar.gz file.'), - 'Stage': -1 - }, - status=415 - ) - - temp_filepath = course_dir / filename - if not course_dir.isdir(): - os.mkdir(course_dir) - - logging.debug('importing course to {0}'.format(temp_filepath)) - - # Get upload chunks byte ranges - try: - matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"]) - content_range = matches.groupdict() - except KeyError: # Single chunk - # no Content-Range header, so make one that will work - content_range = {'start': 0, 'stop': 1, 'end': 2} - - # stream out the uploaded files in chunks to disk - if int(content_range['start']) == 0: - mode = "wb+" - else: - mode = "ab+" - size = os.path.getsize(temp_filepath) - # Check to make sure we haven't missed a chunk - # This shouldn't happen, even if different instances are handling - # the same session, but it's always better to catch errors earlier. - if size < int(content_range['start']): - _save_request_status(request, courselike_string, -1) - log.warning( - "Reported range %s does not match size downloaded so far %s", - content_range['start'], - size - ) - return JsonResponse( - { - 'ErrMsg': _('File upload corrupted. Please try again'), - 'Stage': -1 - }, - status=409 - ) - # The last request sometimes comes twice. This happens because - # nginx sends a 499 error code when the response takes too long. - elif size > int(content_range['stop']) and size == int(content_range['end']): - return JsonResponse({'ImportStatus': 1}) - - with open(temp_filepath, mode) as temp_file: - for chunk in request.FILES['course-data'].chunks(): - temp_file.write(chunk) - - size = os.path.getsize(temp_filepath) - - if int(content_range['stop']) != int(content_range['end']) - 1: - # More chunks coming - return JsonResponse({ - "files": [{ - "name": filename, - "size": size, - "deleteUrl": "", - "deleteType": "", - "url": reverse_course_url('import_handler', courselike_key), - "thumbnailUrl": "" - }] - }) - # Send errors to client with stage at which error occurred. - except Exception as exception: # pylint: disable=broad-except - _save_request_status(request, courselike_string, -1) - if course_dir.isdir(): - shutil.rmtree(course_dir) - log.info("Course import %s: Temp data cleared", courselike_key) - - log.exception( - "error importing course" - ) - return JsonResponse( - { - 'ErrMsg': str(exception), - 'Stage': -1 - }, - status=400 - ) - - # try-finally block for proper clean up after receiving last chunk. - try: - # This was the last chunk. - log.info("Course import %s: Upload complete", courselike_key) - _save_request_status(request, courselike_string, 1) - - tar_file = tarfile.open(temp_filepath) - try: - safetar_extractall(tar_file, (course_dir + '/').encode('utf-8')) - except SuspiciousOperation as exc: - _save_request_status(request, courselike_string, -1) - return JsonResponse( - { - 'ErrMsg': 'Unsafe tar file. Aborting import.', - 'SuspiciousFileOperationMsg': exc.args[0], - 'Stage': -1 - }, - status=400 - ) - finally: - tar_file.close() - - log.info("Course import %s: Uploaded file extracted", courselike_key) - _save_request_status(request, courselike_string, 2) - - # find the 'course.xml' file - def get_all_files(directory): - """ - For each file in the directory, yield a 2-tuple of (file-name, - directory-path) - """ - for dirpath, _dirnames, filenames in os.walk(directory): - for filename in filenames: - yield (filename, dirpath) - - def get_dir_for_fname(directory, filename): - """ - Returns the dirpath for the first file found in the directory - with the given name. If there is no file in the directory with - the specified name, return None. - """ - for fname, dirpath in get_all_files(directory): - if fname == filename: - return dirpath - return None - - dirpath = get_dir_for_fname(course_dir, root_name) - if not dirpath: - _save_request_status(request, courselike_string, -2) - return JsonResponse( - { - 'ErrMsg': _('Could not find the {0} file in the package.').format(root_name), - 'Stage': -2 - }, - status=415 - ) - - dirpath = os.path.relpath(dirpath, data_root) - logging.debug('found %s at %s', root_name, dirpath) - - log.info("Course import %s: Extracted file verified", courselike_key) - _save_request_status(request, courselike_string, 3) - - with dog_stats_api.timer( - 'courselike_import.time', - tags=[u"courselike:{}".format(courselike_key)] - ): - courselike_items = import_func( - modulestore(), request.user.id, - settings.GITHUB_REPO_ROOT, [dirpath], - load_error_modules=False, - static_content_store=contentstore(), - target_id=courselike_key - ) - - new_location = courselike_items[0].location - logging.debug('new course at %s', new_location) - - log.info("Course import %s: Course import successful", courselike_key) - _save_request_status(request, courselike_string, 4) - - # Send errors to client with stage at which error occurred. - except Exception as exception: # pylint: disable=broad-except - log.exception( - "error importing course" - ) - return JsonResponse( - { - 'ErrMsg': str(exception), - 'Stage': -session_status[courselike_string] - }, - status=400 - ) - - finally: - if course_dir.isdir(): - shutil.rmtree(course_dir) - log.info("Course import %s: Temp data cleared", courselike_key) - # set failed stage number with negative sign in case of unsuccessful import - if session_status[courselike_string] != 4: - _save_request_status(request, courselike_string, -abs(session_status[courselike_string])) - - # status == 4 represents that course has been imported successfully. - if session_status[courselike_string] == 4 and root_name == COURSE_ROOT: - # Reload the course so we have the latest state - course = modulestore().get_course(courselike_key) - if course.entrance_exam_enabled: - entrance_exam_chapter = modulestore().get_items( - course.id, - qualifiers={'category': 'chapter'}, - settings={'is_entrance_exam': True} - )[0] - - metadata = {'entrance_exam_id': unicode(entrance_exam_chapter.location)} - CourseMetadata.update_from_dict(metadata, course, request.user) - add_entrance_exam_milestone(course.id, entrance_exam_chapter) - log.info("Course %s Entrance exam imported", course.id) - - return JsonResponse({'Status': 'OK'}) + return _write_chunk(request, courselike_key) elif request.method == 'GET': # assume html status_url = reverse_course_url( "import_status_handler", courselike_key, kwargs={'filename': "fillerName"} @@ -358,6 +110,122 @@ def _save_request_status(request, key, status): request.session.save() +def _write_chunk(request, courselike_key): + """ + Write the OLX file data chunk from the given request to the local filesystem. + """ + # Upload .tar.gz to local filesystem for one-server installations not using S3 or Swift + data_root = path(settings.GITHUB_REPO_ROOT) + subdir = base64.urlsafe_b64encode(repr(courselike_key)) + course_dir = data_root / subdir + filename = request.FILES['course-data'].name + + courselike_string = text_type(courselike_key) + filename + # Do everything in a try-except block to make sure everything is properly cleaned up. + try: + # Use sessions to keep info about import progress + _save_request_status(request, courselike_string, 0) + + if not filename.endswith('.tar.gz'): + _save_request_status(request, courselike_string, -1) + return JsonResponse( + { + 'ErrMsg': _('We only support uploading a .tar.gz file.'), + 'Stage': -1 + }, + status=415 + ) + + temp_filepath = course_dir / filename + if not course_dir.isdir(): # pylint: disable=no-value-for-parameter + os.mkdir(course_dir) + + logging.debug('importing course to {0}'.format(temp_filepath)) + + # Get upload chunks byte ranges + try: + matches = CONTENT_RE.search(request.META["HTTP_CONTENT_RANGE"]) + content_range = matches.groupdict() + except KeyError: # Single chunk + # no Content-Range header, so make one that will work + content_range = {'start': 0, 'stop': 1, 'end': 2} + + # stream out the uploaded files in chunks to disk + if int(content_range['start']) == 0: + mode = "wb+" + else: + mode = "ab+" + size = os.path.getsize(temp_filepath) + # Check to make sure we haven't missed a chunk + # This shouldn't happen, even if different instances are handling + # the same session, but it's always better to catch errors earlier. + if size < int(content_range['start']): + _save_request_status(request, courselike_string, -1) + log.warning( + "Reported range %s does not match size downloaded so far %s", + content_range['start'], + size + ) + return JsonResponse( + { + 'ErrMsg': _('File upload corrupted. Please try again'), + 'Stage': -1 + }, + status=409 + ) + # The last request sometimes comes twice. This happens because + # nginx sends a 499 error code when the response takes too long. + elif size > int(content_range['stop']) and size == int(content_range['end']): + return JsonResponse({'ImportStatus': 1}) + + with open(temp_filepath, mode) as temp_file: + for chunk in request.FILES['course-data'].chunks(): + temp_file.write(chunk) + + size = os.path.getsize(temp_filepath) + + if int(content_range['stop']) != int(content_range['end']) - 1: + # More chunks coming + return JsonResponse({ + "files": [{ + "name": filename, + "size": size, + "deleteUrl": "", + "deleteType": "", + "url": reverse_course_url('import_handler', courselike_key), + "thumbnailUrl": "" + }] + }) + + log.info("Course import %s: Upload complete", courselike_key) + with open(temp_filepath, 'rb') as local_file: + django_file = File(local_file) + storage_path = course_import_export_storage.save(u'olx_import/' + filename, django_file) + import_olx.delay( + request.user.id, text_type(courselike_key), storage_path, filename, request.LANGUAGE_CODE) + + # Send errors to client with stage at which error occurred. + except Exception as exception: # pylint: disable=broad-except + _save_request_status(request, courselike_string, -1) + if course_dir.isdir(): # pylint: disable=no-value-for-parameter + shutil.rmtree(course_dir) + log.info("Course import %s: Temp data cleared", courselike_key) + + log.exception( + "error importing course" + ) + return JsonResponse( + { + 'ErrMsg': str(exception), + 'Stage': -1 + }, + status=400 + ) + + return JsonResponse({'ImportStatus': 1}) + + +@transaction.non_atomic_requests @require_GET @ensure_csrf_cookie @login_required @@ -368,9 +236,9 @@ def import_status_handler(request, course_key_string, filename=None): -X : Import unsuccessful due to some error with X as stage [0-3] 0 : No status info found (import done or upload still in progress) - 1 : Extracting file - 2 : Validating. - 3 : Importing to mongo + 1 : Unpacking + 2 : Verifying + 3 : Updating 4 : Import successful """ @@ -378,87 +246,45 @@ def import_status_handler(request, course_key_string, filename=None): if not has_course_author_access(request.user, course_key): raise PermissionDenied() - try: - session_status = request.session["import_status"] - status = session_status[course_key_string + filename] - except KeyError: - status = 0 - - return JsonResponse({"ImportStatus": status}) - - -def create_export_tarball(course_module, course_key, context): - """ - Generates the export tarball, or returns None if there was an error. - - Updates the context with any error information if applicable. - """ - name = course_module.url_name - export_file = NamedTemporaryFile(prefix=name + '.', suffix=".tar.gz") - root_dir = path(mkdtemp()) - - try: - if isinstance(course_key, LibraryLocator): - export_library_to_xml(modulestore(), contentstore(), course_key, root_dir, name) - else: - export_course_to_xml(modulestore(), contentstore(), course_module.id, root_dir, name) - - logging.debug(u'tar file being generated at %s', export_file.name) - with tarfile.open(name=export_file.name, mode='w:gz') as tar_file: - tar_file.add(root_dir / name, arcname=name) - - except SerializationError as exc: - log.exception(u'There was an error exporting %s', course_key) - unit = None - failed_item = None - parent = None + # The task status record is authoritative once it's been created + args = {u'course_key_string': course_key_string, u'archive_name': filename} + name = CourseImportTask.generate_name(args) + task_status = UserTaskStatus.objects.filter(name=name) + for status_filter in STATUS_FILTERS: + task_status = status_filter().filter_queryset(request, task_status, import_status_handler) + task_status = task_status.order_by(u'-created').first() + if task_status is None: + # The task hasn't been initialized yet; did we store info in the session already? try: - failed_item = modulestore().get_item(exc.location) - parent_loc = modulestore().get_parent_location(failed_item.location) - - if parent_loc is not None: - parent = modulestore().get_item(parent_loc) - if parent.location.category == 'vertical': - unit = parent - except: # pylint: disable=bare-except - # if we have a nested exception, then we'll show the more generic error message - pass - - context.update({ - 'in_err': True, - 'raw_err_msg': str(exc), - 'failed_module': failed_item, - 'unit': unit, - 'edit_unit_url': reverse_usage_url("container_handler", parent.location) if parent else "", - }) - raise - except Exception as exc: - log.exception('There was an error exporting %s', course_key) - context.update({ - 'in_err': True, - 'unit': None, - 'raw_err_msg': str(exc)}) - raise - finally: - shutil.rmtree(root_dir / name) + session_status = request.session["import_status"] + status = session_status[course_key_string + filename] + except KeyError: + status = 0 + elif task_status.state == UserTaskStatus.SUCCEEDED: + status = 4 + elif task_status.state in (UserTaskStatus.FAILED, UserTaskStatus.CANCELED): + status = max(-(task_status.completed_steps + 1), -3) + else: + status = min(task_status.completed_steps + 1, 3) - return export_file + return JsonResponse({"ImportStatus": status}) -def send_tarball(tarball): +def send_tarball(tarball, size): """ Renders a tarball to response, for use when sending a tar.gz file to the user. """ wrapper = FileWrapper(tarball) response = HttpResponse(wrapper, content_type='application/x-tgz') response['Content-Disposition'] = 'attachment; filename=%s' % os.path.basename(tarball.name.encode('utf-8')) - response['Content-Length'] = os.path.getsize(tarball.name) + response['Content-Length'] = size return response +@transaction.non_atomic_requests @ensure_csrf_cookie @login_required -@require_http_methods(("GET",)) +@require_http_methods(('GET', 'POST')) @ensure_valid_course_key def export_handler(request, course_key_string): """ @@ -466,17 +292,14 @@ def export_handler(request, course_key_string): GET html: return html page for import page - application/x-tgz: return tar.gz file containing exported course json: not supported + POST + Start a Celery task to export the course - Note that there are 2 ways to request the tar.gz file. The request header can specify - application/x-tgz via HTTP_ACCEPT, or a query parameter can be used (?_accept=application/x-tgz). - - If the tar.gz file has been requested but the export operation fails, an HTML page will be returned - which describes the error. + The Studio UI uses a POST request to start the export asynchronously, with + a link appearing on the page once it's ready. """ course_key = CourseKey.from_string(course_key_string) - export_url = reverse_course_url('export_handler', course_key) if not has_course_author_access(request.user, course_key): raise PermissionDenied() @@ -496,22 +319,129 @@ def export_handler(request, course_key_string): 'courselike_home_url': reverse_course_url("course_handler", course_key), 'library': False } - - context['export_url'] = export_url + '?_accept=application/x-tgz' + context['status_url'] = reverse_course_url('export_status_handler', course_key) # an _accept URL parameter will be preferred over HTTP_ACCEPT in the header. requested_format = request.GET.get('_accept', request.META.get('HTTP_ACCEPT', 'text/html')) - if 'application/x-tgz' in requested_format: - try: - tarball = create_export_tarball(courselike_module, course_key, context) - except SerializationError: - return render_to_response('export.html', context) - return send_tarball(tarball) - + if request.method == 'POST': + export_olx.delay(request.user.id, course_key_string, request.LANGUAGE_CODE) + return JsonResponse({'ExportStatus': 1}) elif 'text/html' in requested_format: return render_to_response('export.html', context) - else: - # Only HTML or x-tgz request formats are supported (no JSON). + # Only HTML request format is supported (no JSON). return HttpResponse(status=406) + + +@transaction.non_atomic_requests +@require_GET +@ensure_csrf_cookie +@login_required +@ensure_valid_course_key +def export_status_handler(request, course_key_string): + """ + Returns an integer corresponding to the status of a file export. These are: + + -X : Export unsuccessful due to some error with X as stage [0-3] + 0 : No status info found (export done or task not yet created) + 1 : Exporting + 2 : Compressing + 3 : Export successful + + If the export was successful, a URL for the generated .tar.gz file is also + returned. + """ + course_key = CourseKey.from_string(course_key_string) + if not has_course_author_access(request.user, course_key): + raise PermissionDenied() + + # The task status record is authoritative once it's been created + task_status = _latest_task_status(request, course_key_string, export_status_handler) + output_url = None + error = None + if task_status is None: + # The task hasn't been initialized yet; did we store info in the session already? + try: + session_status = request.session["export_status"] + status = session_status[course_key_string] + except KeyError: + status = 0 + elif task_status.state == UserTaskStatus.SUCCEEDED: + status = 3 + artifact = UserTaskArtifact.objects.get(status=task_status, name='Output') + if hasattr(artifact.file.storage, 'bucket'): + filename = os.path.basename(artifact.file.name).encode('utf-8') + disposition = 'attachment; filename="{}"'.format(filename) + output_url = artifact.file.storage.url(artifact.file.name, response_headers={ + 'response-content-disposition': disposition, + 'response-content-encoding': 'application/octet-stream', + 'response-content-type': 'application/x-tgz' + }) + else: + # local file, serve from the authorization wrapper view + output_url = reverse_course_url('export_output_handler', course_key) + + elif task_status.state in (UserTaskStatus.FAILED, UserTaskStatus.CANCELED): + status = max(-(task_status.completed_steps + 1), -2) + errors = UserTaskArtifact.objects.filter(status=task_status, name='Error') + if len(errors): + error = errors[0].text + try: + error = json.loads(error) + except ValueError: + # Wasn't JSON, just use the value as a string + pass + else: + status = min(task_status.completed_steps + 1, 2) + + response = {"ExportStatus": status} + if output_url: + response['ExportOutput'] = output_url + elif error: + response['ExportError'] = error + return JsonResponse(response) + + +@transaction.non_atomic_requests +@require_GET +@ensure_csrf_cookie +@login_required +@ensure_valid_course_key +def export_output_handler(request, course_key_string): + """ + Returns the OLX .tar.gz produced by a file export. Only used in + environments such as devstack where the output is stored in a local + filesystem instead of an external service like S3. + """ + course_key = CourseKey.from_string(course_key_string) + if not has_course_author_access(request.user, course_key): + raise PermissionDenied() + + task_status = _latest_task_status(request, course_key_string, export_output_handler) + if task_status and task_status.state == UserTaskStatus.SUCCEEDED: + artifact = None + try: + artifact = UserTaskArtifact.objects.get(status=task_status, name='Output') + tarball = course_import_export_storage.open(artifact.file.name) + return send_tarball(tarball, artifact.file.storage.size(artifact.file.name)) + except UserTaskArtifact.DoesNotExist: + raise Http404 + finally: + if artifact: + artifact.file.close() + else: + raise Http404 + + +def _latest_task_status(request, course_key_string, view_func=None): + """ + Get the most recent export status update for the specified course/library + key. + """ + args = {u'course_key_string': course_key_string} + name = CourseExportTask.generate_name(args) + task_status = UserTaskStatus.objects.filter(name=name) + for status_filter in STATUS_FILTERS: + task_status = status_filter().filter_queryset(request, task_status, view_func) + return task_status.order_by(u'-created').first() diff --git a/cms/djangoapps/contentstore/views/item.py b/cms/djangoapps/contentstore/views/item.py index f6f924f3ea3d..b4452f6f9806 100644 --- a/cms/djangoapps/contentstore/views/item.py +++ b/cms/djangoapps/contentstore/views/item.py @@ -8,50 +8,62 @@ from functools import partial from uuid import uuid4 -import dogstats_wrapper as dog_stats_api from django.conf import settings from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User from django.core.exceptions import PermissionDenied -from django.http import HttpResponseBadRequest, HttpResponse, Http404 +from django.http import Http404, HttpResponse, HttpResponseBadRequest from django.utils.translation import ugettext as _ from django.views.decorators.http import require_http_methods from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import LibraryUsageLocator from pytz import UTC +from xblock.core import XBlock from xblock.fields import Scope from xblock.fragment import Fragment -from xblock_django.user_service import DjangoXBlockUserService +import dogstats_wrapper as dog_stats_api from cms.lib.xblock.authoring_mixin import VISIBILITY_VIEW from contentstore.utils import ( - find_release_date_source, find_staff_lock_source, is_currently_visible_to_students, - ancestor_has_staff_lock, has_children_visible_to_specific_content_groups, + ancestor_has_staff_lock, + find_release_date_source, + find_staff_lock_source, + get_split_group_display_name, get_user_partition_info, + has_children_visible_to_specific_partition_groups, + is_currently_visible_to_students, + is_self_paced +) +from contentstore.views.helpers import ( + create_xblock, + get_parent_xblock, + is_unit, + usage_key_with_run, + xblock_primary_child_category, + xblock_studio_url, + xblock_type_display_name ) -from contentstore.views.helpers import is_unit, xblock_studio_url, xblock_primary_child_category, \ - xblock_type_display_name, get_parent_xblock, create_xblock, usage_key_with_run from contentstore.views.preview import get_preview_fragment -from contentstore.utils import is_self_paced - -from openedx.core.lib.gating import api as gating_api from edxmako.shortcuts import render_to_string from models.settings.course_grading import CourseGradingModel -from openedx.core.lib.xblock_utils import wrap_xblock, request_token +from openedx.core.lib.gating import api as gating_api +from openedx.core.lib.xblock_utils import request_token, wrap_xblock from static_replace import replace_static_urls -from student.auth import has_studio_write_access, has_studio_read_access +from student.auth import has_studio_read_access, has_studio_write_access from util.date_utils import get_default_time_display -from util.json_request import expect_json, JsonResponse +from util.json_request import JsonResponse, expect_json from util.milestones_helpers import is_entrance_exams_enabled +from xblock_config.models import CourseEditLTIFieldsEnabledFlag +from xblock_django.user_service import DjangoXBlockUserService from xmodule.course_module import DEFAULT_START_DATE -from xmodule.modulestore import ModuleStoreEnum, EdxJSONEncoder +from xmodule.modulestore import EdxJSONEncoder, ModuleStoreEnum from xmodule.modulestore.django import modulestore from xmodule.modulestore.draft_and_published import DIRECT_ONLY_CATEGORIES -from xmodule.modulestore.exceptions import ItemNotFoundError, InvalidLocationError +from xmodule.modulestore.exceptions import InvalidLocationError, ItemNotFoundError from xmodule.modulestore.inheritance import own_metadata +from xmodule.services import ConfigurationService, SettingsService from xmodule.tabs import CourseTabList -from xmodule.x_module import PREVIEW_VIEWS, STUDIO_VIEW, STUDENT_VIEW, DEPRECATION_VSCOMPAT_EVENT - +from xmodule.x_module import DEPRECATION_VSCOMPAT_EVENT, PREVIEW_VIEWS, STUDENT_VIEW, STUDIO_VIEW __all__ = [ 'orphan_handler', 'xblock_handler', 'xblock_view_handler', 'xblock_outline_handler', 'xblock_container_handler' @@ -98,6 +110,7 @@ def xblock_handler(request, usage_key_string): GET json: returns representation of the xblock (locator id, data, and metadata). if ?fields=graderType, it returns the graderType for the unit instead of the above. + if ?fields=ancestorInfo, it returns ancestor info of the xblock. html: returns HTML for rendering the xblock (which includes both the "preview" view and the "editor" view) PUT or POST or PATCH json: if xblock locator is specified, update the xblock instance. The json payload can contain @@ -126,8 +139,11 @@ def xblock_handler(request, usage_key_string): if usage_key_string is not specified, create a new xblock instance, either by duplicating an existing xblock, or creating an entirely new one. The json playload can contain these fields: - :parent_locator: parent for new xblock, required for both duplicate and create new instance + :parent_locator: parent for new xblock, required for duplicate, move and create new instance :duplicate_source_locator: if present, use this as the source for creating a duplicate copy + :move_source_locator: if present, use this as the source item for moving + :target_index: if present, use this as the target index for moving an item to a particular index + otherwise target_index is calculated. It is sent back in the response. :category: type of xblock, required if duplicate_source_locator is not present. :display_name: name for new xblock, optional :boilerplate: template name for populating fields, optional and only used @@ -149,6 +165,10 @@ def xblock_handler(request, usage_key_string): if 'graderType' in fields: # right now can't combine output of this w/ output of _get_module_info, but worthy goal return JsonResponse(CourseGradingModel.get_section_grader_type(usage_key)) + elif 'ancestorInfo' in fields: + xblock = _get_xblock(usage_key, request.user) + ancestor_info = _create_xblock_ancestor_info(xblock, is_concise=True) + return JsonResponse(ancestor_info) # TODO: pass fields to _get_module_info and only return those with modulestore().bulk_operations(usage_key.course_key): response = _get_module_info(_get_xblock(usage_key, request.user)) @@ -193,14 +213,26 @@ def xblock_handler(request, usage_key_string): request.user, request.json.get('display_name'), ) - - return JsonResponse({"locator": unicode(dest_usage_key), "courseKey": unicode(dest_usage_key.course_key)}) + return JsonResponse({'locator': unicode(dest_usage_key), 'courseKey': unicode(dest_usage_key.course_key)}) else: return _create_item(request) + elif request.method == 'PATCH': + if 'move_source_locator' in request.json: + move_source_usage_key = usage_key_with_run(request.json.get('move_source_locator')) + target_parent_usage_key = usage_key_with_run(request.json.get('parent_locator')) + target_index = request.json.get('target_index') + if ( + not has_studio_write_access(request.user, target_parent_usage_key.course_key) or + not has_studio_read_access(request.user, target_parent_usage_key.course_key) + ): + raise PermissionDenied() + return _move_item(move_source_usage_key, target_parent_usage_key, request.user, target_index) + + return JsonResponse({'error': 'Patch request did not recognise any parameters to handle.'}, status=400) else: return HttpResponseBadRequest( - "Only instance creation is supported without a usage key.", - content_type="text/plain" + 'Only instance creation is supported without a usage key.', + content_type='text/plain' ) @@ -244,6 +276,10 @@ def service(self, block, service_name): return DjangoXBlockUserService(self._user) if service_name == "studio_user_permissions": return StudioPermissionsService(self._user) + if service_name == "settings": + return SettingsService() + if service_name == "lti-configuration": + return ConfigurationService(CourseEditLTIFieldsEnabledFlag) return None @@ -631,9 +667,140 @@ def _create_item(request): ) return JsonResponse( - {"locator": unicode(created_block.location), "courseKey": unicode(created_block.location.course_key)} + {'locator': unicode(created_block.location), 'courseKey': unicode(created_block.location.course_key)} + ) + + +def _get_source_index(source_usage_key, source_parent): + """ + Get source index position of the XBlock. + + Arguments: + source_usage_key (BlockUsageLocator): Locator of source item. + source_parent (XBlock): A parent of the source XBlock. + + Returns: + source_index (int): Index position of the xblock in a parent. + """ + try: + source_index = source_parent.children.index(source_usage_key) + return source_index + except ValueError: + return None + + +def is_source_item_in_target_parents(source_item, target_parent): + """ + Returns True if source item is found in target parents otherwise False. + + Arguments: + source_item (XBlock): Source Xblock. + target_parent (XBlock): Target XBlock. + """ + target_ancestors = _create_xblock_ancestor_info(target_parent, is_concise=True)['ancestors'] + for target_ancestor in target_ancestors: + if unicode(source_item.location) == target_ancestor['id']: + return True + return False + + +def _move_item(source_usage_key, target_parent_usage_key, user, target_index=None): + """ + Move an existing xblock as a child of the supplied target_parent_usage_key. + + Arguments: + source_usage_key (BlockUsageLocator): Locator of source item. + target_parent_usage_key (BlockUsageLocator): Locator of target parent. + target_index (int): If provided, insert source item at provided index location in target_parent_usage_key item. + + Returns: + JsonResponse: Information regarding move operation. It may contains error info if an invalid move operation + is performed. + """ + # Get the list of all parentable component type XBlocks. + parent_component_types = list( + set(name for name, class_ in XBlock.load_classes() if getattr(class_, 'has_children', False)) - + set(DIRECT_ONLY_CATEGORIES) ) + store = modulestore() + with store.bulk_operations(source_usage_key.course_key): + source_item = store.get_item(source_usage_key) + source_parent = source_item.get_parent() + target_parent = store.get_item(target_parent_usage_key) + source_type = source_item.category + target_parent_type = target_parent.category + error = None + + # Store actual/initial index of the source item. This would be sent back with response, + # so that with Undo operation, it would easier to move back item to it's original/old index. + source_index = _get_source_index(source_usage_key, source_parent) + + valid_move_type = { + 'sequential': 'vertical', + 'chapter': 'sequential', + } + + if (valid_move_type.get(target_parent_type, '') != source_type and + target_parent_type not in parent_component_types): + error = _('You can not move {source_type} into {target_parent_type}.').format( + source_type=source_type, + target_parent_type=target_parent_type, + ) + elif source_parent.location == target_parent.location or source_item.location in target_parent.children: + error = _('Item is already present in target location.') + elif source_item.location == target_parent.location: + error = _('You can not move an item into itself.') + elif is_source_item_in_target_parents(source_item, target_parent): + error = _('You can not move an item into it\'s child.') + elif target_parent_type == 'split_test': + error = _('You can not move an item directly into content experiment.') + elif source_index is None: + error = _('{source_usage_key} not found in {parent_usage_key}.').format( + source_usage_key=unicode(source_usage_key), + parent_usage_key=unicode(source_parent.location) + ) + else: + try: + target_index = int(target_index) if target_index is not None else None + if len(target_parent.children) < target_index: + error = _('You can not move {source_usage_key} at an invalid index ({target_index}).').format( + source_usage_key=unicode(source_usage_key), + target_index=target_index + ) + except ValueError: + error = _('You must provide target_index ({target_index}) as an integer.').format( + target_index=target_index + ) + if error: + return JsonResponse({'error': error}, status=400) + + # When target_index is provided, insert xblock at target_index position, otherwise insert at the end. + insert_at = target_index if target_index is not None else len(target_parent.children) + + store.update_item_parent( + item_location=source_item.location, + new_parent_location=target_parent.location, + old_parent_location=source_parent.location, + insert_at=insert_at, + user_id=user.id + ) + + log.info( + 'MOVE: %s moved from %s to %s at %d index', + unicode(source_usage_key), + unicode(source_parent.location), + unicode(target_parent_usage_key), + insert_at + ) + + context = { + 'move_source_locator': unicode(source_usage_key), + 'parent_locator': unicode(target_parent_usage_key), + 'source_index': target_index if target_index is not None else source_index + } + return JsonResponse(context) + def _duplicate_item(parent_usage_key, duplicate_source_usage_key, user, display_name=None, is_child=False): """ @@ -849,6 +1016,7 @@ def _get_module_info(xblock, rewrite_static_links=True, include_ancestor_info=Fa ) if include_publishing_info: add_container_page_publishing_info(xblock, xblock_info) + return xblock_info @@ -887,7 +1055,7 @@ def _get_gating_info(course, xblock): def create_xblock_info(xblock, data=None, metadata=None, include_ancestor_info=False, include_child_info=False, course_outline=False, include_children_predicate=NEVER, parent_xblock=None, graders=None, - user=None, course=None): + user=None, course=None, is_concise=False): """ Creates the information needed for client-side XBlockInfo. @@ -897,6 +1065,7 @@ def create_xblock_info(xblock, data=None, metadata=None, include_ancestor_info=F There are three optional boolean parameters: include_ancestor_info - if true, ancestor info is added to the response include_child_info - if true, direct child info is included in the response + is_concise - if true, returns the concise version of xblock info, default is false. course_outline - if true, the xblock is being rendered on behalf of the course outline. There are certain expensive computations that do not need to be included in this case. @@ -933,20 +1102,22 @@ def create_xblock_info(xblock, data=None, metadata=None, include_ancestor_info=F graders, include_children_predicate=include_children_predicate, user=user, - course=course + course=course, + is_concise=is_concise ) else: child_info = None release_date = _get_release_date(xblock, user) - if xblock.category != 'course': + if xblock.category != 'course' and not is_concise: visibility_state = _compute_visibility_state( xblock, child_info, is_xblock_unit and has_changes, is_self_paced(course) ) else: visibility_state = None published = modulestore().has_published_version(xblock) if not is_library_block else None + published_on = get_default_time_display(xblock.published_on) if published and xblock.published_on else None # defining the default value 'True' for delete, duplicate, drag and add new child actions # in xblock_actions for each xblock. @@ -970,83 +1141,99 @@ def create_xblock_info(xblock, data=None, metadata=None, include_ancestor_info=F pct_sign=_('%')) xblock_info = { - "id": unicode(xblock.location), - "display_name": xblock.display_name_with_default, - "category": xblock.category, - "edited_on": get_default_time_display(xblock.subtree_edited_on) if xblock.subtree_edited_on else None, - "published": published, - "published_on": get_default_time_display(xblock.published_on) if published and xblock.published_on else None, - "studio_url": xblock_studio_url(xblock, parent_xblock), - "released_to_students": datetime.now(UTC) > xblock.start, - "release_date": release_date, - "visibility_state": visibility_state, - "has_explicit_staff_lock": xblock.fields['visible_to_staff_only'].is_set_on(xblock), - "start": xblock.fields['start'].to_json(xblock.start), - "graded": xblock.graded, - "due_date": get_default_time_display(xblock.due), - "due": xblock.fields['due'].to_json(xblock.due), - "format": xblock.format, - "course_graders": [grader.get('type') for grader in graders], - "has_changes": has_changes, - "actions": xblock_actions, - "explanatory_message": explanatory_message, - "group_access": xblock.group_access, - "user_partitions": get_user_partition_info(xblock, course=course), + 'id': unicode(xblock.location), + 'display_name': xblock.display_name_with_default, + 'category': xblock.category, + 'has_children': xblock.has_children } - - if xblock.category == 'sequential': + if is_concise: + if child_info and len(child_info.get('children', [])) > 0: + xblock_info['child_info'] = child_info + # Groups are labelled with their internal ids, rather than with the group name. Replace id with display name. + group_display_name = get_split_group_display_name(xblock, course) + xblock_info['display_name'] = group_display_name if group_display_name else xblock_info['display_name'] + else: + user_partitions = get_user_partition_info(xblock, course=course) xblock_info.update({ - "hide_after_due": xblock.hide_after_due, + 'edited_on': get_default_time_display(xblock.subtree_edited_on) if xblock.subtree_edited_on else None, + 'published': published, + 'published_on': published_on, + 'studio_url': xblock_studio_url(xblock, parent_xblock), + 'released_to_students': datetime.now(UTC) > xblock.start, + 'release_date': release_date, + 'visibility_state': visibility_state, + 'has_explicit_staff_lock': xblock.fields['visible_to_staff_only'].is_set_on(xblock), + 'start': xblock.fields['start'].to_json(xblock.start), + 'graded': xblock.graded, + 'due_date': get_default_time_display(xblock.due), + 'due': xblock.fields['due'].to_json(xblock.due), + 'format': xblock.format, + 'course_graders': [grader.get('type') for grader in graders], + 'has_changes': has_changes, + 'actions': xblock_actions, + 'explanatory_message': explanatory_message, + 'group_access': xblock.group_access, + 'user_partitions': user_partitions, 'show_correctness': xblock.show_correctness, }) - # update xblock_info with special exam information if the feature flag is enabled - if settings.FEATURES.get('ENABLE_SPECIAL_EXAMS'): - if xblock.category == 'course': + if xblock.category == 'sequential': xblock_info.update({ - "enable_proctored_exams": xblock.enable_proctored_exams, - "create_zendesk_tickets": xblock.create_zendesk_tickets, - "enable_timed_exams": xblock.enable_timed_exams - }) - elif xblock.category == 'sequential': - xblock_info.update({ - "is_proctored_exam": xblock.is_proctored_exam, - "is_practice_exam": xblock.is_practice_exam, - "is_time_limited": xblock.is_time_limited, - "exam_review_rules": xblock.exam_review_rules, - "default_time_limit_minutes": xblock.default_time_limit_minutes, + 'hide_after_due': xblock.hide_after_due, }) - # Update with gating info - xblock_info.update(_get_gating_info(course, xblock)) - - if xblock.category == 'sequential': - # Entrance exam subsection should be hidden. in_entrance_exam is - # inherited metadata, all children will have it. - if getattr(xblock, "in_entrance_exam", False): - xblock_info["is_header_visible"] = False - - if data is not None: - xblock_info["data"] = data - if metadata is not None: - xblock_info["metadata"] = metadata - if include_ancestor_info: - xblock_info['ancestor_info'] = _create_xblock_ancestor_info(xblock, course_outline) - if child_info: - xblock_info['child_info'] = child_info - if visibility_state == VisibilityState.staff_only: - xblock_info["ancestor_has_staff_lock"] = ancestor_has_staff_lock(xblock, parent_xblock) - else: - xblock_info["ancestor_has_staff_lock"] = False + # update xblock_info with special exam information if the feature flag is enabled + if settings.FEATURES.get('ENABLE_SPECIAL_EXAMS'): + if xblock.category == 'course': + xblock_info.update({ + 'enable_proctored_exams': xblock.enable_proctored_exams, + 'create_zendesk_tickets': xblock.create_zendesk_tickets, + 'enable_timed_exams': xblock.enable_timed_exams + }) + elif xblock.category == 'sequential': + xblock_info.update({ + 'is_proctored_exam': xblock.is_proctored_exam, + 'is_practice_exam': xblock.is_practice_exam, + 'is_time_limited': xblock.is_time_limited, + 'exam_review_rules': xblock.exam_review_rules, + 'default_time_limit_minutes': xblock.default_time_limit_minutes, + }) - if course_outline: - if xblock_info["has_explicit_staff_lock"]: - xblock_info["staff_only_message"] = True - elif child_info and child_info["children"]: - xblock_info["staff_only_message"] = all([child["staff_only_message"] for child in child_info["children"]]) + # Update with gating info + xblock_info.update(_get_gating_info(course, xblock)) + + if xblock.category == 'sequential': + # Entrance exam subsection should be hidden. in_entrance_exam is + # inherited metadata, all children will have it. + if getattr(xblock, 'in_entrance_exam', False): + xblock_info['is_header_visible'] = False + + if data is not None: + xblock_info['data'] = data + if metadata is not None: + xblock_info['metadata'] = metadata + if include_ancestor_info: + xblock_info['ancestor_info'] = _create_xblock_ancestor_info(xblock, course_outline, include_child_info=True) + if child_info: + xblock_info['child_info'] = child_info + if visibility_state == VisibilityState.staff_only: + xblock_info['ancestor_has_staff_lock'] = ancestor_has_staff_lock(xblock, parent_xblock) else: - xblock_info["staff_only_message"] = False + xblock_info['ancestor_has_staff_lock'] = False + + if course_outline: + if xblock_info['has_explicit_staff_lock']: + xblock_info['staff_only_message'] = True + elif child_info and child_info['children']: + xblock_info['staff_only_message'] = all( + [child['staff_only_message'] for child in child_info['children']] + ) + else: + xblock_info['staff_only_message'] = False + xblock_info["has_partition_group_components"] = has_children_visible_to_specific_partition_groups( + xblock + ) return xblock_info @@ -1075,7 +1262,7 @@ def safe_get_username(user_id): xblock_info["edited_by"] = safe_get_username(xblock.subtree_edited_by) xblock_info["published_by"] = safe_get_username(xblock.published_by) xblock_info["currently_visible_to_students"] = is_currently_visible_to_students(xblock) - xblock_info["has_content_group_components"] = has_children_visible_to_specific_content_groups(xblock) + xblock_info["has_partition_group_components"] = has_children_visible_to_specific_partition_groups(xblock) if xblock_info["release_date"]: xblock_info["release_date_from"] = _get_release_date_from(xblock) if xblock_info["visibility_state"] == VisibilityState.staff_only: @@ -1156,14 +1343,14 @@ def _compute_visibility_state(xblock, child_info, is_unit_with_changes, is_cours return VisibilityState.ready -def _create_xblock_ancestor_info(xblock, course_outline): +def _create_xblock_ancestor_info(xblock, course_outline=False, include_child_info=False, is_concise=False): """ Returns information about the ancestors of an xblock. Note that the direct parent will also return information about all of its children. """ ancestors = [] - def collect_ancestor_info(ancestor, include_child_info=False): + def collect_ancestor_info(ancestor, include_child_info=False, is_concise=False): """ Collect xblock info regarding the specified xblock and its ancestors. """ @@ -1173,16 +1360,18 @@ def collect_ancestor_info(ancestor, include_child_info=False): ancestor, include_child_info=include_child_info, course_outline=course_outline, - include_children_predicate=direct_children_only + include_children_predicate=direct_children_only, + is_concise=is_concise )) - collect_ancestor_info(get_parent_xblock(ancestor)) - collect_ancestor_info(get_parent_xblock(xblock), include_child_info=True) + collect_ancestor_info(get_parent_xblock(ancestor), is_concise=is_concise) + collect_ancestor_info(get_parent_xblock(xblock), include_child_info=include_child_info, is_concise=is_concise) return { 'ancestors': ancestors } -def _create_xblock_child_info(xblock, course_outline, graders, include_children_predicate=NEVER, user=None, course=None): # pylint: disable=line-too-long +def _create_xblock_child_info(xblock, course_outline, graders, include_children_predicate=NEVER, user=None, + course=None, is_concise=False): # pylint: disable=line-too-long """ Returns information about the children of an xblock, as well as about the primary category of xblock expected as children. @@ -1203,6 +1392,7 @@ def _create_xblock_child_info(xblock, course_outline, graders, include_children_ graders=graders, user=user, course=course, + is_concise=is_concise ) for child in xblock.get_children() ] return child_info diff --git a/cms/djangoapps/contentstore/views/library.py b/cms/djangoapps/contentstore/views/library.py index cb3cd4e18b46..bce8079f6273 100644 --- a/cms/djangoapps/contentstore/views/library.py +++ b/cms/djangoapps/contentstore/views/library.py @@ -7,30 +7,36 @@ import logging -from contentstore.views.item import create_xblock_info -from contentstore.utils import reverse_library_url, add_instructor -from django.http import HttpResponseNotAllowed, Http404 +from django.conf import settings from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied -from django.conf import settings +from django.http import Http404, HttpResponseForbidden, HttpResponseNotAllowed from django.utils.translation import ugettext as _ -from django.views.decorators.http import require_http_methods from django.views.decorators.csrf import ensure_csrf_cookie -from edxmako.shortcuts import render_to_response +from django.views.decorators.http import require_http_methods from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import LibraryLocator, LibraryUsageLocator -from xmodule.modulestore.exceptions import DuplicateCourseError -from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.django import modulestore -from .user import user_with_role -from .component import get_component_templates, CONTAINER_TEMPLATES +from contentstore.utils import add_instructor, reverse_library_url +from contentstore.views.item import create_xblock_info +from course_creators.views import get_course_creator_status +from edxmako.shortcuts import render_to_response from student.auth import ( - STUDIO_VIEW_USERS, STUDIO_EDIT_ROLES, get_user_permissions, has_studio_read_access, has_studio_write_access + STUDIO_EDIT_ROLES, + STUDIO_VIEW_USERS, + get_user_permissions, + has_studio_read_access, + has_studio_write_access ) from student.roles import CourseInstructorRole, CourseStaffRole, LibraryUserRole -from util.json_request import expect_json, JsonResponse, JsonResponseBadRequest +from util.json_request import JsonResponse, JsonResponseBadRequest, expect_json +from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import DuplicateCourseError + +from .component import CONTAINER_TEMPLATES, get_component_templates +from .user import user_with_role __all__ = ['library_handler', 'manage_library_users'] @@ -39,6 +45,22 @@ LIBRARIES_ENABLED = settings.FEATURES.get('ENABLE_CONTENT_LIBRARIES', False) +def get_library_creator_status(user): + """ + Helper method for returning the library creation status for a particular user, + taking into account the value LIBRARIES_ENABLED. + """ + + if not LIBRARIES_ENABLED: + return False + elif user.is_staff: + return True + elif settings.FEATURES.get('ENABLE_CREATOR_GROUP', False): + return get_course_creator_status(user) == 'granted' + else: + return True + + @login_required @ensure_csrf_cookie @require_http_methods(('GET', 'POST')) @@ -50,6 +72,10 @@ def library_handler(request, library_key_string=None): log.exception("Attempted to use the content library API when the libraries feature is disabled.") raise Http404 # Should never happen because we test the feature in urls.py also + if not get_library_creator_status(request.user): + if not request.user.is_staff: + return HttpResponseForbidden() + if library_key_string is not None and request.method == 'POST': return HttpResponseNotAllowed(("POST",)) diff --git a/cms/djangoapps/contentstore/views/organization.py b/cms/djangoapps/contentstore/views/organization.py index 6c530dfe4fdd..253229963a14 100644 --- a/cms/djangoapps/contentstore/views/organization.py +++ b/cms/djangoapps/contentstore/views/organization.py @@ -1,8 +1,8 @@ """Organizations views for use with Studio.""" from django.contrib.auth.decorators import login_required +from django.http import HttpResponse from django.utils.decorators import method_decorator from django.views.generic import View -from django.http import HttpResponse from openedx.core.djangolib.js_utils import dump_js_escaped_json from util.organizations_helpers import get_organizations diff --git a/cms/djangoapps/contentstore/views/preview.py b/cms/djangoapps/contentstore/views/preview.py index 7c146e099ef9..c33dd463800f 100644 --- a/cms/djangoapps/contentstore/views/preview.py +++ b/cms/djangoapps/contentstore/views/preview.py @@ -4,41 +4,45 @@ from functools import partial from django.conf import settings +from django.contrib.auth.decorators import login_required from django.core.urlresolvers import reverse from django.http import Http404, HttpResponseBadRequest -from django.contrib.auth.decorators import login_required -from edxmako.shortcuts import render_to_string +from django.utils.translation import ugettext as _ +from opaque_keys.edx.keys import UsageKey +from xblock.django.request import django_to_webob_request, webob_to_django_response +from xblock.exceptions import NoSuchHandlerError +from xblock.fragment import Fragment +from xblock.runtime import KvsFieldData +import static_replace +from cms.lib.xblock.field_data import CmsFieldData +from contentstore.utils import get_visibility_partition_info +from contentstore.views.access import get_user_role +from edxmako.shortcuts import render_to_string +from lms.djangoapps.lms_xblock.field_data import LmsFieldData +from openedx.core.lib.license import wrap_with_license from openedx.core.lib.xblock_utils import ( - replace_static_urls, wrap_xblock, wrap_fragment, wrap_xblock_aside, request_token, xblock_local_resource_url, + replace_static_urls, + request_token, + wrap_fragment, + wrap_xblock, + wrap_xblock_aside, + xblock_local_resource_url ) -from xmodule.x_module import PREVIEW_VIEWS, STUDENT_VIEW, AUTHOR_VIEW +from util.sandboxing import can_execute_unsafe_code, get_python_lib_zip +from xblock_config.models import StudioConfig +from xblock_django.user_service import DjangoXBlockUserService from xmodule.contentstore.django import contentstore from xmodule.error_module import ErrorDescriptor from xmodule.exceptions import NotFoundError, ProcessingError -from xmodule.studio_editable import has_author_view +from xmodule.modulestore.django import ModuleI18nService, modulestore +from xmodule.partitions.partitions_service import PartitionService from xmodule.services import SettingsService -from xmodule.modulestore.django import modulestore, ModuleI18nService -from xmodule.mixin import wrap_with_license -from opaque_keys.edx.keys import UsageKey -from xmodule.x_module import ModuleSystem -from xblock.runtime import KvsFieldData -from xblock.django.request import webob_to_django_response, django_to_webob_request -from xblock.exceptions import NoSuchHandlerError -from xblock.fragment import Fragment -from xblock_django.user_service import DjangoXBlockUserService - -from lms.djangoapps.lms_xblock.field_data import LmsFieldData -from cms.lib.xblock.field_data import CmsFieldData - -from util.sandboxing import can_execute_unsafe_code, get_python_lib_zip +from xmodule.studio_editable import has_author_view +from xmodule.x_module import AUTHOR_VIEW, PREVIEW_VIEWS, STUDENT_VIEW, ModuleSystem -import static_replace -from .session_kv_store import SessionKeyValueStore from .helpers import render_from_lms - -from contentstore.views.access import get_user_role -from xblock_config.models import StudioConfig +from .session_kv_store import SessionKeyValueStore __all__ = ['preview_handler'] @@ -92,9 +96,6 @@ class PreviewModuleSystem(ModuleSystem): # pylint: disable=abstract-method # they are being rendered for preview (i.e. in Studio) is_author_mode = True - def __init__(self, **kwargs): - super(PreviewModuleSystem, self).__init__(**kwargs) - def handler_url(self, block, handler_name, suffix='', query='', thirdparty=False): return reverse('preview_handler', kwargs={ 'usage_key_string': unicode(block.scope_ids.usage_id), @@ -213,10 +214,24 @@ def _preview_module_system(request, descriptor, field_data): "i18n": ModuleI18nService, "settings": SettingsService(), "user": DjangoXBlockUserService(request.user), + "partitions": StudioPartitionService(course_id=course_id) }, ) +class StudioPartitionService(PartitionService): + """ + A runtime mixin to allow the display and editing of component visibility based on user partitions. + """ + def get_user_group_id_for_partition(self, user, user_partition_id): + """ + Override this method to return None, as the split_test_module calls this + to determine which group a user should see, but is robust to getting a return + value of None meaning that all groups should be shown. + """ + return None + + def _load_preview_module(request, descriptor): """ Return a preview XModule instantiated from the supplied descriptor. Will use mutable fields @@ -264,6 +279,10 @@ def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False): root_xblock = context.get('root_xblock') is_root = root_xblock and xblock.location == root_xblock.location is_reorderable = _is_xblock_reorderable(xblock, context) + selected_groups_label = get_visibility_partition_info(xblock)['selected_groups_label'] + if selected_groups_label: + selected_groups_label = _('Access restricted to: {list_of_groups}').format(list_of_groups=selected_groups_label) + course = modulestore().get_course(xblock.location.course_key) template_context = { 'xblock_context': context, 'xblock': xblock, @@ -273,8 +292,12 @@ def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False): 'is_reorderable': is_reorderable, 'can_edit': context.get('can_edit', True), 'can_edit_visibility': context.get('can_edit_visibility', True), + 'selected_groups_label': selected_groups_label, 'can_add': context.get('can_add', True), + 'can_move': context.get('can_move', True), + 'language': getattr(course, 'language', None) } + html = render_to_string('studio_xblock_wrapper.html', template_context) frag = wrap_fragment(frag, html) return frag diff --git a/cms/djangoapps/contentstore/views/program.py b/cms/djangoapps/contentstore/views/program.py deleted file mode 100644 index 0ac51d5df0ab..000000000000 --- a/cms/djangoapps/contentstore/views/program.py +++ /dev/null @@ -1,67 +0,0 @@ -"""Programs views for use with Studio.""" -from django.conf import settings -from django.contrib.auth.decorators import login_required -from django.core.exceptions import ImproperlyConfigured -from django.core.urlresolvers import reverse -from django.http import Http404, JsonResponse -from django.utils.decorators import method_decorator -from django.views.generic import View -from provider.oauth2.models import Client - -from edxmako.shortcuts import render_to_response -from openedx.core.djangoapps.programs.models import ProgramsApiConfig -from openedx.core.lib.token_utils import JwtBuilder - - -class ProgramAuthoringView(View): - """View rendering a template which hosts the Programs authoring app. - - The Programs authoring app is a Backbone SPA. The app handles its own routing - and provides a UI which can be used to create and publish new Programs. - """ - - @method_decorator(login_required) - def get(self, request, *args, **kwargs): - """Populate the template context with values required for the authoring app to run.""" - programs_config = ProgramsApiConfig.current() - - if programs_config.is_studio_tab_enabled and request.user.is_staff: - return render_to_response('program_authoring.html', { - 'lms_base_url': '//{}/'.format(settings.LMS_BASE), - 'programs_api_url': programs_config.public_api_url, - 'programs_token_url': reverse('programs_id_token'), - 'studio_home_url': reverse('home'), - 'uses_pattern_library': True - }) - else: - raise Http404 - - -class ProgramsIdTokenView(View): - """Provides id tokens to JavaScript clients for use with the Programs API.""" - - @method_decorator(login_required) - def get(self, request, *args, **kwargs): - """Generate and return a token, if the integration is enabled.""" - if ProgramsApiConfig.current().is_studio_tab_enabled: - # TODO: Use the system's JWT_AUDIENCE and JWT_SECRET_KEY instead of client ID and name. - client_name = 'programs' - - try: - client = Client.objects.get(name=client_name) - except Client.DoesNotExist: - raise ImproperlyConfigured( - 'OAuth2 Client with name [{}] does not exist.'.format(client_name) - ) - - scopes = ['email', 'profile'] - expires_in = settings.OAUTH_ID_TOKEN_EXPIRATION - jwt = JwtBuilder(request.user, secret=client.client_secret).build_token( - scopes, - expires_in, - aud=client.client_id - ) - - return JsonResponse({'id_token': jwt}) - else: - raise Http404 diff --git a/cms/djangoapps/contentstore/views/public.py b/cms/djangoapps/contentstore/views/public.py index 5aeb8c88bdb7..e423ecc3f7f8 100644 --- a/cms/djangoapps/contentstore/views/public.py +++ b/cms/djangoapps/contentstore/views/public.py @@ -1,20 +1,15 @@ """ Public views """ -from django.views.decorators.csrf import ensure_csrf_cookie -from django.views.decorators.clickjacking import xframe_options_deny +from django.conf import settings from django.core.context_processors import csrf from django.core.urlresolvers import reverse from django.shortcuts import redirect -from django.conf import settings +from django.views.decorators.clickjacking import xframe_options_deny +from django.views.decorators.csrf import ensure_csrf_cookie from edxmako.shortcuts import render_to_response - -from openedx.core.djangoapps.external_auth.views import ( - ssl_login_shortcut, - ssl_get_cert_from_request, - redirect_with_get, -) +from openedx.core.djangoapps.external_auth.views import redirect_with_get, ssl_get_cert_from_request, ssl_login_shortcut from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers __all__ = ['signup', 'login_page', 'howitworks'] diff --git a/cms/djangoapps/contentstore/views/tabs.py b/cms/djangoapps/contentstore/views/tabs.py index f79d4d07211b..a05f4f79d1a5 100644 --- a/cms/djangoapps/contentstore/views/tabs.py +++ b/cms/djangoapps/contentstore/views/tabs.py @@ -1,20 +1,19 @@ """ Views related to course tabs """ -from student.auth import has_course_author_access -from util.json_request import expect_json, JsonResponse - -from django.http import HttpResponseNotFound from django.contrib.auth.decorators import login_required from django.core.exceptions import PermissionDenied +from django.http import HttpResponseNotFound from django.views.decorators.csrf import ensure_csrf_cookie from django.views.decorators.http import require_http_methods +from opaque_keys.edx.keys import CourseKey, UsageKey from edxmako.shortcuts import render_to_response -from xmodule.modulestore.django import modulestore +from student.auth import has_course_author_access +from util.json_request import JsonResponse, expect_json from xmodule.modulestore import ModuleStoreEnum -from xmodule.tabs import CourseTabList, CourseTab, InvalidTabsException, StaticTab -from opaque_keys.edx.keys import CourseKey, UsageKey +from xmodule.modulestore.django import modulestore +from xmodule.tabs import CourseTab, CourseTabList, InvalidTabsException, StaticTab from ..utils import get_lms_link_for_item diff --git a/cms/djangoapps/contentstore/views/tests/test_access.py b/cms/djangoapps/contentstore/views/tests/test_access.py index c38a292ba7a6..235193b98027 100644 --- a/cms/djangoapps/contentstore/views/tests/test_access.py +++ b/cms/djangoapps/contentstore/views/tests/test_access.py @@ -1,14 +1,14 @@ """ Tests access.py """ -from django.test import TestCase from django.contrib.auth.models import User +from django.test import TestCase +from opaque_keys.edx.locations import SlashSeparatedCourseKey +from contentstore.views.access import get_user_role +from student.auth import add_users from student.roles import CourseInstructorRole, CourseStaffRole from student.tests.factories import AdminFactory -from student.auth import add_users -from contentstore.views.access import get_user_role -from opaque_keys.edx.locations import SlashSeparatedCourseKey class RolesTest(TestCase): diff --git a/cms/djangoapps/contentstore/views/tests/test_assets.py b/cms/djangoapps/contentstore/views/tests/test_assets.py index 7848f63335e6..57fe05d06410 100644 --- a/cms/djangoapps/contentstore/views/tests/test_assets.py +++ b/cms/djangoapps/contentstore/views/tests/test_assets.py @@ -1,29 +1,29 @@ """ Unit tests for the asset upload endpoint. """ +import json from datetime import datetime from io import BytesIO -from pytz import UTC -from PIL import Image -import json -from mock import patch + +import mock +from ddt import data, ddt from django.conf import settings +from django.test.utils import override_settings +from mock import patch +from opaque_keys.edx.locations import AssetLocation, SlashSeparatedCourseKey +from PIL import Image +from pytz import UTC from contentstore.tests.utils import CourseTestCase -from contentstore.views import assets from contentstore.utils import reverse_course_url +from contentstore.views import assets +from static_replace import replace_static_urls from xmodule.assetstore import AssetMetadata from xmodule.contentstore.content import StaticContent from xmodule.contentstore.django import contentstore -from xmodule.modulestore.django import modulestore from xmodule.modulestore import ModuleStoreEnum +from xmodule.modulestore.django import modulestore from xmodule.modulestore.xml_importer import import_course_from_xml -from django.test.utils import override_settings -from opaque_keys.edx.locations import SlashSeparatedCourseKey, AssetLocation -from static_replace import replace_static_urls -import mock -from ddt import ddt -from ddt import data TEST_DATA_DIR = settings.COMMON_TEST_DATA_ROOT diff --git a/cms/djangoapps/contentstore/views/tests/test_certificates.py b/cms/djangoapps/contentstore/views/tests/test_certificates.py index 3215aaaf2615..26bbc9be1872 100644 --- a/cms/djangoapps/contentstore/views/tests/test_certificates.py +++ b/cms/djangoapps/contentstore/views/tests/test_certificates.py @@ -5,28 +5,24 @@ """ import itertools import json -import mock -import ddt +import ddt +import mock from django.conf import settings from django.test.utils import override_settings - from opaque_keys.edx.keys import AssetKey -from contentstore.utils import reverse_course_url -from contentstore.views.certificates import CERTIFICATE_SCHEMA_VERSION from contentstore.tests.utils import CourseTestCase -from xmodule.contentstore.django import contentstore -from xmodule.contentstore.content import StaticContent -from xmodule.exceptions import NotFoundError +from contentstore.utils import get_lms_link_for_certificate_web_view, reverse_course_url +from contentstore.views.certificates import CERTIFICATE_SCHEMA_VERSION, CertificateManager +from course_modes.tests.factories import CourseModeFactory from student.models import CourseEnrollment from student.roles import CourseInstructorRole, CourseStaffRole from student.tests.factories import UserFactory -from course_modes.tests.factories import CourseModeFactory -from contentstore.views.certificates import CertificateManager -from django.test.utils import override_settings -from contentstore.utils import get_lms_link_for_certificate_web_view from util.testing import EventTestMixin, UrlResetMixin +from xmodule.contentstore.content import StaticContent +from xmodule.contentstore.django import contentstore +from xmodule.exceptions import NotFoundError FEATURES_WITH_CERTS_ENABLED = settings.FEATURES.copy() FEATURES_WITH_CERTS_ENABLED['CERTIFICATES_HTML_VIEW'] = True diff --git a/cms/djangoapps/contentstore/views/tests/test_container_page.py b/cms/djangoapps/contentstore/views/tests/test_container_page.py index b22e06327fe6..76f0c186fabd 100644 --- a/cms/djangoapps/contentstore/views/tests/test_container_page.py +++ b/cms/djangoapps/contentstore/views/tests/test_container_page.py @@ -1,22 +1,24 @@ """ Unit tests for the container page. """ -import re import datetime -from pytz import UTC -from mock import patch, Mock +import re from django.http import Http404 from django.test.client import RequestFactory from django.utils import http +from mock import Mock, patch +from pytz import UTC import contentstore.views.component as views +from contentstore.tests.test_libraries import LibraryTestCase from contentstore.views.tests.utils import StudioPageTestCase +from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.factories import ItemFactory +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory -class ContainerPageTestCase(StudioPageTestCase): +class ContainerPageTestCase(StudioPageTestCase, LibraryTestCase): """ Unit tests for the container page. """ @@ -128,6 +130,44 @@ def test_public_container_preview_html(self): self.validate_preview_html(published_child_container, self.container_view) self.validate_preview_html(published_child_vertical, self.reorderable_child_view) + def test_library_page_preview_html(self): + """ + Verify that a library xblock's container (library page) preview returns the expected HTML. + """ + # Add some content to library. + self._add_simple_content_block() + self.validate_preview_html(self.library, self.container_view, can_reorder=False, can_move=False) + + def test_library_content_preview_html(self): + """ + Verify that a library content block container page preview returns the expected HTML. + """ + # Library content block is only supported in split courses. + with modulestore().default_store(ModuleStoreEnum.Type.split): + course = CourseFactory.create() + + # Add some content to library + self._add_simple_content_block() + + # Create a library content block + lc_block = self._add_library_content_block(course, self.lib_key) + self.assertEqual(len(lc_block.children), 0) + + # Refresh children to be reflected in lc_block + lc_block = self._refresh_children(lc_block) + self.assertEqual(len(lc_block.children), 1) + + self.validate_preview_html( + lc_block, + self.container_view, + can_add=False, + can_reorder=False, + can_move=False, + can_edit=True, + can_duplicate=False, + can_delete=False + ) + def test_draft_container_preview_html(self): """ Verify that a draft xblock's container preview returns the expected HTML. diff --git a/cms/djangoapps/contentstore/views/tests/test_course_index.py b/cms/djangoapps/contentstore/views/tests/test_course_index.py index 70b3e6387f1e..83636e0e6bc8 100644 --- a/cms/djangoapps/contentstore/views/tests/test_course_index.py +++ b/cms/djangoapps/contentstore/views/tests/test_course_index.py @@ -1,34 +1,37 @@ """ Unit tests for getting the list of courses and the course outline. """ -import ddt +import datetime import json + +import ddt import lxml -import datetime import mock import pytz - from django.conf import settings from django.core.exceptions import PermissionDenied from django.utils.translation import ugettext as _ +from opaque_keys.edx.locator import CourseLocator +from search.api import perform_search from contentstore.courseware_index import CoursewareSearchIndexer, SearchIndexingError from contentstore.tests.utils import CourseTestCase -from contentstore.utils import reverse_course_url, reverse_library_url, add_instructor, reverse_usage_url +from contentstore.utils import add_instructor, reverse_course_url, reverse_library_url, reverse_usage_url from contentstore.views.course import ( - course_outline_initial_state, reindex_course_and_check_access, _deprecated_blocks_info + _deprecated_blocks_info, + course_outline_initial_state, + reindex_course_and_check_access ) -from contentstore.views.item import create_xblock_info, VisibilityState +from contentstore.views.item import VisibilityState, create_xblock_info from course_action_state.managers import CourseRerunUIStateManager from course_action_state.models import CourseRerunState -from opaque_keys.edx.locator import CourseLocator -from search.api import perform_search from student.auth import has_course_author_access +from student.roles import LibraryUserRole from student.tests.factories import UserFactory from util.date_utils import get_default_time_display from xmodule.modulestore import ModuleStoreEnum -from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, LibraryFactory @@ -75,22 +78,37 @@ def test_libraries_on_course_index(self): """ Test getting the list of libraries from the course listing page """ + def _assert_library_link_present(response, library): + """ + Asserts there's a valid library link on libraries tab. + """ + parsed_html = lxml.html.fromstring(response.content) + library_link_elements = parsed_html.find_class('library-link') + self.assertEqual(len(library_link_elements), 1) + link = library_link_elements[0] + self.assertEqual( + link.get("href"), + reverse_library_url('library_handler', library.location.library_key), + ) + # now test that url + outline_response = self.client.get(link.get("href"), {}, HTTP_ACCEPT='text/html') + self.assertEqual(outline_response.status_code, 200) + # Add a library: lib1 = LibraryFactory.create() index_url = '/home/' index_response = self.client.get(index_url, {}, HTTP_ACCEPT='text/html') - parsed_html = lxml.html.fromstring(index_response.content) - library_link_elements = parsed_html.find_class('library-link') - self.assertEqual(len(library_link_elements), 1) - link = library_link_elements[0] - self.assertEqual( - link.get("href"), - reverse_library_url('library_handler', lib1.location.library_key), - ) - # now test that url - outline_response = self.client.get(link.get("href"), {}, HTTP_ACCEPT='text/html') - self.assertEqual(outline_response.status_code, 200) + _assert_library_link_present(index_response, lib1) + + # Make sure libraries are visible to non-staff users too + self.client.logout() + non_staff_user, non_staff_userpassword = self.create_non_staff_user() + lib2 = LibraryFactory.create(user_id=non_staff_user.id) + LibraryUserRole(lib2.location.library_key).add_users(non_staff_user) + self.client.login(username=non_staff_user.username, password=non_staff_userpassword) + index_response = self.client.get(index_url, {}, HTTP_ACCEPT='text/html') + _assert_library_link_present(index_response, lib2) def test_is_staff_access(self): """ @@ -315,6 +333,8 @@ class TestCourseOutline(CourseTestCase): """ Unit tests for the course outline. """ + ENABLED_SIGNALS = ['course_published'] + def setUp(self): """ Set up the for the course outline tests. @@ -334,11 +354,16 @@ def setUp(self): parent_location=self.vertical.location, category="video", display_name="My Video" ) - def test_json_responses(self): + @ddt.data(True, False) + def test_json_responses(self, is_concise): """ Verify the JSON responses returned for the course. + + Arguments: + is_concise (Boolean) : If True, fetch concise version of course outline. """ outline_url = reverse_course_url('course_handler', self.course.id) + outline_url = outline_url + '?format=concise' if is_concise else outline_url resp = self.client.get(outline_url, HTTP_ACCEPT='application/json') json_response = json.loads(resp.content) @@ -346,8 +371,8 @@ def test_json_responses(self): self.assertEqual(json_response['category'], 'course') self.assertEqual(json_response['id'], unicode(self.course.location)) self.assertEqual(json_response['display_name'], self.course.display_name) - self.assertTrue(json_response['published']) - self.assertIsNone(json_response['visibility_state']) + self.assertNotEqual(json_response.get('published', False), is_concise) + self.assertIsNone(json_response.get('visibility_state')) # Now verify the first child children = json_response['child_info']['children'] @@ -356,24 +381,25 @@ def test_json_responses(self): self.assertEqual(first_child_response['category'], 'chapter') self.assertEqual(first_child_response['id'], unicode(self.chapter.location)) self.assertEqual(first_child_response['display_name'], 'Week 1') - self.assertTrue(json_response['published']) - self.assertEqual(first_child_response['visibility_state'], VisibilityState.unscheduled) + self.assertNotEqual(json_response.get('published', False), is_concise) + if not is_concise: + self.assertEqual(first_child_response['visibility_state'], VisibilityState.unscheduled) self.assertGreater(len(first_child_response['child_info']['children']), 0) # Finally, validate the entire response for consistency - self.assert_correct_json_response(json_response) + self.assert_correct_json_response(json_response, is_concise) - def assert_correct_json_response(self, json_response): + def assert_correct_json_response(self, json_response, is_concise=False): """ Asserts that the JSON response is syntactically consistent """ self.assertIsNotNone(json_response['display_name']) self.assertIsNotNone(json_response['id']) self.assertIsNotNone(json_response['category']) - self.assertTrue(json_response['published']) + self.assertNotEqual(json_response.get('published', False), is_concise) if json_response.get('child_info', None): for child_response in json_response['child_info']['children']: - self.assert_correct_json_response(child_response) + self.assert_correct_json_response(child_response, is_concise) def test_course_outline_initial_state(self): course_module = modulestore().get_item(self.course.location) @@ -456,10 +482,9 @@ def _verify_deprecated_info(self, course_id, advanced_modules, info, deprecated_ ] ) - self.assertEqual(info['block_types'], deprecated_block_types) self.assertEqual( - info['block_types_enabled'], - any(component in advanced_modules for component in deprecated_block_types) + info['deprecated_enabled_block_types'], + [component for component in advanced_modules if component in deprecated_block_types] ) self.assertItemsEqual(info['blocks'], expected_blocks) @@ -488,6 +513,28 @@ def test_verify_deprecated_warning_message(self, publish, block_types): block_types ) + @ddt.data( + (["a", "b", "c"], ["a", "b", "c"]), + (["a", "b", "c"], ["a", "b", "d"]), + (["a", "b", "c"], ["a", "d", "e"]), + (["a", "b", "c"], ["d", "e", "f"]) + ) + @ddt.unpack + def test_verify_warn_only_on_enabled_modules(self, enabled_block_types, deprecated_block_types): + """ + Verify that we only warn about block_types that are both deprecated and enabled. + """ + expected_block_types = list(set(enabled_block_types) & set(deprecated_block_types)) + course_module = modulestore().get_item(self.course.location) + self._create_test_data(course_module, create_blocks=True, block_types=enabled_block_types) + info = _deprecated_blocks_info(course_module, deprecated_block_types) + self._verify_deprecated_info( + course_module.id, + course_module.advanced_modules, + info, + expected_block_types + ) + @ddt.data( {'delete_vertical': True}, {'delete_vertical': False}, @@ -563,6 +610,8 @@ class TestCourseReIndex(CourseTestCase): """ SUCCESSFUL_RESPONSE = _("Course has been successfully reindexed.") + ENABLED_SIGNALS = ['course_published'] + def setUp(self): """ Set up the for the course outline tests. diff --git a/cms/djangoapps/contentstore/views/tests/test_course_updates.py b/cms/djangoapps/contentstore/views/tests/test_course_updates.py index 1687941a22ca..1a1069f003b8 100644 --- a/cms/djangoapps/contentstore/views/tests/test_course_updates.py +++ b/cms/djangoapps/contentstore/views/tests/test_course_updates.py @@ -2,13 +2,14 @@ unit tests for course_info views and models. """ import json -from mock import patch + from django.test.utils import override_settings +from mock import patch +from opaque_keys.edx.keys import UsageKey from contentstore.models import PushNotificationConfig from contentstore.tests.test_course_settings import CourseTestCase from contentstore.utils import reverse_course_url, reverse_usage_url -from opaque_keys.edx.keys import UsageKey from xmodule.modulestore.django import modulestore diff --git a/cms/djangoapps/contentstore/views/tests/test_credit_eligibility.py b/cms/djangoapps/contentstore/views/tests/test_credit_eligibility.py index a93e7d864180..6f882a79d102 100644 --- a/cms/djangoapps/contentstore/views/tests/test_credit_eligibility.py +++ b/cms/djangoapps/contentstore/views/tests/test_credit_eligibility.py @@ -6,15 +6,15 @@ from contentstore.tests.utils import CourseTestCase from contentstore.utils import reverse_course_url -from xmodule.modulestore.tests.factories import CourseFactory - from openedx.core.djangoapps.credit.api import get_credit_requirements from openedx.core.djangoapps.credit.models import CreditCourse from openedx.core.djangoapps.credit.signals import on_course_publish +from xmodule.modulestore.tests.factories import CourseFactory class CreditEligibilityTest(CourseTestCase): - """Base class to test the course settings details view in Studio for credit + """ + Base class to test the course settings details view in Studio for credit eligibility requirements. """ def setUp(self): @@ -24,7 +24,8 @@ def setUp(self): @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_CREDIT_ELIGIBILITY': False}) def test_course_details_with_disabled_setting(self): - """Test that user don't see credit eligibility requirements in response + """ + Test that user don't see credit eligibility requirements in response if the feature flag 'ENABLE_CREDIT_ELIGIBILITY' is not enabled. """ response = self.client.get_html(self.course_details_url) @@ -34,7 +35,8 @@ def test_course_details_with_disabled_setting(self): @mock.patch.dict("django.conf.settings.FEATURES", {'ENABLE_CREDIT_ELIGIBILITY': True}) def test_course_details_with_enabled_setting(self): - """Test that credit eligibility requirements are present in + """ + Test that credit eligibility requirements are present in response if the feature flag 'ENABLE_CREDIT_ELIGIBILITY' is enabled. """ # verify that credit eligibility requirements block don't show if the diff --git a/cms/djangoapps/contentstore/views/tests/test_entrance_exam.py b/cms/djangoapps/contentstore/views/tests/test_entrance_exam.py index fbf2d770bd72..a20122aade35 100644 --- a/cms/djangoapps/contentstore/views/tests/test_entrance_exam.py +++ b/cms/djangoapps/contentstore/views/tests/test_entrance_exam.py @@ -2,25 +2,29 @@ Test module for Entrance Exams AJAX callback handler workflows """ import json -from mock import patch from django.conf import settings from django.contrib.auth.models import User from django.test.client import RequestFactory +from milestones.tests.utils import MilestonesTestCaseMixin +from mock import patch +from opaque_keys.edx.keys import UsageKey from contentstore.tests.utils import AjaxEnabledTestClient, CourseTestCase from contentstore.utils import reverse_url -from contentstore.views.entrance_exam import create_entrance_exam, update_entrance_exam, delete_entrance_exam,\ - add_entrance_exam_milestone, remove_entrance_exam_milestone_reference -from contentstore.views.helpers import GRADER_TYPES +from contentstore.views.entrance_exam import ( + add_entrance_exam_milestone, + create_entrance_exam, + delete_entrance_exam, + remove_entrance_exam_milestone_reference, + update_entrance_exam +) +from contentstore.views.helpers import GRADER_TYPES, create_xblock from models.settings.course_grading import CourseGradingModel from models.settings.course_metadata import CourseMetadata -from opaque_keys.edx.keys import UsageKey from student.tests.factories import UserFactory from util import milestones_helpers from xmodule.modulestore.django import modulestore -from contentstore.views.helpers import create_xblock -from milestones.tests.utils import MilestonesTestCaseMixin @patch.dict(settings.FEATURES, {'ENTRANCE_EXAMS': True}) diff --git a/cms/djangoapps/contentstore/views/tests/test_gating.py b/cms/djangoapps/contentstore/views/tests/test_gating.py index 5ef6438ba280..ebbbf3444048 100644 --- a/cms/djangoapps/contentstore/views/tests/test_gating.py +++ b/cms/djangoapps/contentstore/views/tests/test_gating.py @@ -4,12 +4,13 @@ import json from mock import patch -from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE -from xmodule.modulestore.tests.factories import ItemFactory + from contentstore.tests.utils import CourseTestCase from contentstore.utils import reverse_usage_url from contentstore.views.item import VisibilityState from openedx.core.lib.gating.api import GATING_NAMESPACE_QUALIFIER +from xmodule.modulestore.tests.django_utils import TEST_DATA_SPLIT_MODULESTORE +from xmodule.modulestore.tests.factories import ItemFactory class TestSubsectionGating(CourseTestCase): @@ -17,6 +18,7 @@ class TestSubsectionGating(CourseTestCase): Tests for the subsection gating feature """ MODULESTORE = TEST_DATA_SPLIT_MODULESTORE + ENABLED_SIGNALS = ['item_deleted'] def setUp(self): """ @@ -127,8 +129,8 @@ def test_get_prerequisite(self, mock_is_prereq, mock_get_required_content, mock_ self.assertEqual(resp['prereq_min_score'], 100) self.assertEqual(resp['visibility_state'], VisibilityState.gated) - @patch('contentstore.signals.gating_api.set_required_content') - @patch('contentstore.signals.gating_api.remove_prerequisite') + @patch('contentstore.signals.handlers.gating_api.set_required_content') + @patch('contentstore.signals.handlers.gating_api.remove_prerequisite') def test_delete_item_signal_handler_called(self, mock_remove_prereq, mock_set_required): seq3 = ItemFactory.create( parent_location=self.chapter.location, diff --git a/cms/djangoapps/contentstore/views/tests/test_group_configurations.py b/cms/djangoapps/contentstore/views/tests/test_group_configurations.py index 5e04336d606d..ff2710c3ef35 100644 --- a/cms/djangoapps/contentstore/views/tests/test_group_configurations.py +++ b/cms/djangoapps/contentstore/views/tests/test_group_configurations.py @@ -8,7 +8,7 @@ from mock import patch from contentstore.utils import reverse_course_url, reverse_usage_url -from contentstore.course_group_config import GroupConfiguration +from contentstore.course_group_config import GroupConfiguration, CONTENT_GROUP_CONFIGURATION_NAME from contentstore.tests.utils import CourseTestCase from xmodule.partitions.partitions import Group, UserPartition from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory @@ -210,12 +210,6 @@ class GroupConfigurationsListHandlerTestCase(CourseTestCase, GroupConfigurations """ Test cases for group_configurations_list_handler. """ - def setUp(self): - """ - Set up GroupConfigurationsListHandlerTestCase. - """ - super(GroupConfigurationsListHandlerTestCase, self).setUp() - def _url(self): """ Return url for the handler. @@ -240,7 +234,7 @@ def test_view_index_ok(self): self.assertEqual(response.status_code, 200) self.assertContains(response, 'First name') self.assertContains(response, 'Group C') - self.assertContains(response, 'Content Group Configuration') + self.assertContains(response, CONTENT_GROUP_CONFIGURATION_NAME) def test_unsupported_http_accept_header(self): """ @@ -609,9 +603,14 @@ class GroupConfigurationsUsageInfoTestCase(CourseTestCase, HelperMethods): """ Tests for usage information of configurations and content groups. """ - - def setUp(self): - super(GroupConfigurationsUsageInfoTestCase, self).setUp() + def _get_user_partition(self, scheme): + """ + Returns the first user partition with the specified scheme. + """ + for group in GroupConfiguration.get_all_user_partition_details(self.store, self.course): + if group['scheme'] == scheme: + return group + return None def _get_expected_content_group(self, usage_for_group): """ @@ -637,7 +636,7 @@ def test_content_group_not_used(self): Test that right data structure will be created if content group is not used. """ self._add_user_partitions(scheme_id='cohort') - actual = GroupConfiguration.get_or_create_content_group(self.store, self.course) + actual = self._get_user_partition('cohort') expected = self._get_expected_content_group(usage_for_group=[]) self.assertEqual(actual, expected) @@ -650,7 +649,7 @@ def test_can_get_correct_usage_info_when_special_characters_are_in_content(self) cid=0, group_id=1, name_suffix='0', special_characters=u"JOSÉ ANDRÉS" ) - actual = GroupConfiguration.get_or_create_content_group(self.store, self.course) + actual = self._get_user_partition('cohort') expected = self._get_expected_content_group( usage_for_group=[ { @@ -669,7 +668,7 @@ def test_can_get_correct_usage_info_for_content_groups(self): self._add_user_partitions(count=1, scheme_id='cohort') vertical, __ = self._create_problem_with_content_group(cid=0, group_id=1, name_suffix='0') - actual = GroupConfiguration.get_or_create_content_group(self.store, self.course) + actual = self._get_user_partition('cohort') expected = self._get_expected_content_group(usage_for_group=[ { @@ -706,7 +705,7 @@ def test_can_get_correct_usage_info_with_orphan(self, module_store_type): expected = self._get_expected_content_group(usage_for_group=[]) # Get the actual content group information - actual = GroupConfiguration.get_or_create_content_group(self.store, self.course) + actual = self._get_user_partition('cohort') # Assert that actual content group information is same as expected one. self.assertEqual(actual, expected) @@ -720,7 +719,7 @@ def test_can_use_one_content_group_in_multiple_problems(self): vertical, __ = self._create_problem_with_content_group(cid=0, group_id=1, name_suffix='0') vertical1, __ = self._create_problem_with_content_group(cid=0, group_id=1, name_suffix='1') - actual = GroupConfiguration.get_or_create_content_group(self.store, self.course) + actual = self._get_user_partition('cohort') expected = self._get_expected_content_group(usage_for_group=[ { @@ -927,7 +926,7 @@ def test_can_handle_multiple_partitions(self): # This used to cause an exception since the code assumed that # only one partition would be available. - actual = GroupConfiguration.get_content_groups_usage_info(self.store, self.course) + actual = GroupConfiguration.get_partitions_usage_info(self.store, self.course) self.assertEqual(actual.keys(), [0]) actual = GroupConfiguration.get_content_groups_items_usage_info(self.store, self.course) @@ -938,9 +937,6 @@ class GroupConfigurationsValidationTestCase(CourseTestCase, HelperMethods): """ Tests for validation in Group Configurations. """ - def setUp(self): - super(GroupConfigurationsValidationTestCase, self).setUp() - @patch('xmodule.split_test_module.SplitTestDescriptor.validate_split_test') def verify_validation_add_usage_info(self, expected_result, mocked_message, mocked_validation_messages): """ diff --git a/cms/djangoapps/contentstore/views/tests/test_helpers.py b/cms/djangoapps/contentstore/views/tests/test_helpers.py index 576ea388081d..ace156d281ed 100644 --- a/cms/djangoapps/contentstore/views/tests/test_helpers.py +++ b/cms/djangoapps/contentstore/views/tests/test_helpers.py @@ -2,10 +2,11 @@ Unit tests for helpers.py. """ +from django.utils import http + from contentstore.tests.utils import CourseTestCase from contentstore.views.helpers import xblock_studio_url, xblock_type_display_name from xmodule.modulestore.tests.factories import ItemFactory, LibraryFactory -from django.utils import http class HelpersTestCase(CourseTestCase): diff --git a/cms/djangoapps/contentstore/views/tests/test_import_export.py b/cms/djangoapps/contentstore/views/tests/test_import_export.py index ddbee107bf4d..28eb68a7e996 100644 --- a/cms/djangoapps/contentstore/views/tests/test_import_export.py +++ b/cms/djangoapps/contentstore/views/tests/test_import_export.py @@ -2,41 +2,37 @@ Unit tests for course import and export """ import copy -import ddt import json import logging -import lxml import os import shutil import tarfile import tempfile -from path import Path as path from uuid import uuid4 -from django.test.utils import override_settings +import ddt +import lxml from django.conf import settings +from django.test.utils import override_settings +from milestones.tests.utils import MilestonesTestCaseMixin +from opaque_keys.edx.locator import LibraryLocator +from path import Path as path from contentstore.tests.test_libraries import LibraryTestCase -from xmodule.contentstore.django import contentstore -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.xml_exporter import export_library_to_xml, export_course_to_xml -from xmodule.modulestore.xml_importer import import_library_from_xml, import_course_from_xml -from xmodule.modulestore import LIBRARY_ROOT, ModuleStoreEnum -from contentstore.utils import reverse_course_url from contentstore.tests.utils import CourseTestCase - -from xmodule.modulestore.tests.factories import ItemFactory, LibraryFactory, CourseFactory -from xmodule.modulestore.tests.utils import ( - MongoContentstoreBuilder, SPLIT_MODULESTORE_SETUP, TEST_DATA_DIR -) -from opaque_keys.edx.locator import LibraryLocator - +from contentstore.utils import reverse_course_url +from models.settings.course_metadata import CourseMetadata from openedx.core.lib.extract_tar import safetar_extractall from student import auth from student.roles import CourseInstructorRole, CourseStaffRole -from models.settings.course_metadata import CourseMetadata from util import milestones_helpers -from milestones.tests.utils import MilestonesTestCaseMixin +from xmodule.contentstore.django import contentstore +from xmodule.modulestore import LIBRARY_ROOT, ModuleStoreEnum +from xmodule.modulestore.django import modulestore +from xmodule.modulestore.tests.factories import CourseFactory, ItemFactory, LibraryFactory +from xmodule.modulestore.tests.utils import SPLIT_MODULESTORE_SETUP, TEST_DATA_DIR, MongoContentstoreBuilder +from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml +from xmodule.modulestore.xml_importer import import_course_from_xml, import_library_from_xml TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex @@ -184,7 +180,7 @@ def test_no_coursexml(self): "name": self.bad_tar, "course-data": [btar] }) - self.assertEquals(resp.status_code, 415) + self.assertEquals(resp.status_code, 200) # Check that `import_status` returns the appropriate stage (i.e., the # stage at which import failed). resp_status = self.client.get( @@ -336,8 +332,16 @@ def try_tar(tarpath): with open(tarpath) as tar: args = {"name": tarpath, "course-data": [tar]} resp = self.client.post(self.url, args) - self.assertEquals(resp.status_code, 400) - self.assertIn("SuspiciousFileOperation", resp.content) + self.assertEquals(resp.status_code, 200) + resp = self.client.get( + reverse_course_url( + 'import_status_handler', + self.course.id, + kwargs={'filename': os.path.split(tarpath)[1]} + ) + ) + status = json.loads(resp.content)["ImportStatus"] + self.assertEqual(status, -1) try_tar(self._fifo_tar()) try_tar(self._symlink_tar()) @@ -523,6 +527,7 @@ def setUp(self): """ super(ExportTestCase, self).setUp() self.url = reverse_course_url('export_handler', self.course.id) + self.status_url = reverse_course_url('export_status_handler', self.course.id) def test_export_html(self): """ @@ -539,18 +544,19 @@ def test_export_json_unsupported(self): resp = self.client.get(self.url, HTTP_ACCEPT='application/json') self.assertEquals(resp.status_code, 406) - def test_export_targz(self): + def test_export_async(self): """ - Get tar.gz file, using HTTP_ACCEPT. + Get tar.gz file, using asynchronous background task """ - resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz') - self._verify_export_succeeded(resp) - - def test_export_targz_urlparam(self): - """ - Get tar.gz file, using URL parameter. - """ - resp = self.client.get(self.url + '?_accept=application/x-tgz') + resp = self.client.post(self.url) + self.assertEquals(resp.status_code, 200) + resp = self.client.get(self.status_url) + result = json.loads(resp.content) + status = result['ExportStatus'] + self.assertEquals(status, 3) + self.assertIn('ExportOutput', result) + output_url = result['ExportOutput'] + resp = self.client.get(output_url) self._verify_export_succeeded(resp) def _verify_export_succeeded(self, resp): @@ -580,11 +586,16 @@ def test_export_failure_subsection_level(self): def _verify_export_failure(self, expected_text): """ Export failure helper method. """ - resp = self.client.get(self.url, HTTP_ACCEPT='application/x-tgz') + resp = self.client.post(self.url) self.assertEquals(resp.status_code, 200) - self.assertIsNone(resp.get('Content-Disposition')) - self.assertContains(resp, 'Unable to create xml for module') - self.assertContains(resp, expected_text) + resp = self.client.get(self.status_url) + self.assertEquals(resp.status_code, 200) + result = json.loads(resp.content) + self.assertNotIn('ExportOutput', result) + self.assertIn('ExportError', result) + error = result['ExportError'] + self.assertIn('Unable to create xml for module', error['raw_error_msg']) + self.assertIn(expected_text, error['edit_unit_url']) def test_library_export(self): """ @@ -631,19 +642,53 @@ def test_export_success_with_custom_tag(self): data=xml_string ) - self.test_export_targz_urlparam() + self.test_export_async() @ddt.data( '/export/non.1/existence_1/Run_1', # For mongo '/export/course-v1:non1+existence1+Run1', # For split ) - def test_export_course_doest_not_exist(self, url): + def test_export_course_does_not_exist(self, url): """ - Export failure if course is not exist + Export failure if course does not exist """ resp = self.client.get_html(url) self.assertEquals(resp.status_code, 404) + def test_non_course_author(self): + """ + Verify that users who aren't authors of the course are unable to export it + """ + client, _ = self.create_non_staff_authed_user_client() + resp = client.get(self.url) + self.assertEqual(resp.status_code, 403) + + def test_status_non_course_author(self): + """ + Verify that users who aren't authors of the course are unable to see the status of export tasks + """ + client, _ = self.create_non_staff_authed_user_client() + resp = client.get(self.status_url) + self.assertEqual(resp.status_code, 403) + + def test_status_missing_record(self): + """ + Attempting to get the status of an export task which isn't currently + represented in the database should yield a useful result + """ + resp = self.client.get(self.status_url) + self.assertEqual(resp.status_code, 200) + result = json.loads(resp.content) + self.assertEqual(result['ExportStatus'], 0) + + def test_output_non_course_author(self): + """ + Verify that users who aren't authors of the course are unable to see the output of export tasks + """ + client, _ = self.create_non_staff_authed_user_client() + resp = client.get(reverse_course_url('export_output_handler', self.course.id)) + self.assertEqual(resp.status_code, 403) + @override_settings(CONTENTSTORE=TEST_DATA_CONTENTSTORE) class TestLibraryImportExport(CourseTestCase): diff --git a/cms/djangoapps/contentstore/views/tests/test_item.py b/cms/djangoapps/contentstore/views/tests/test_item.py index baf4ed8faa17..3867a6f72a06 100644 --- a/cms/djangoapps/contentstore/views/tests/test_item.py +++ b/cms/djangoapps/contentstore/views/tests/test_item.py @@ -8,19 +8,22 @@ from pyquery import PyQuery from webob import Response +from django.conf import settings from django.http import Http404 from django.test import TestCase from django.test.client import RequestFactory from django.core.urlresolvers import reverse from contentstore.utils import reverse_usage_url, reverse_course_url +from opaque_keys import InvalidKeyError from openedx.core.djangoapps.self_paced.models import SelfPacedConfiguration from contentstore.views.component import ( component_handler, get_component_templates ) from contentstore.views.item import ( - create_xblock_info, ALWAYS, VisibilityState, _xblock_type_and_display_name, add_container_page_publishing_info + create_xblock_info, _get_source_index, _get_module_info, ALWAYS, VisibilityState, _xblock_type_and_display_name, + add_container_page_publishing_info ) from contentstore.tests.utils import CourseTestCase from student.tests.factories import UserFactory @@ -28,6 +31,7 @@ from xmodule.capa_module import CapaDescriptor from xmodule.modulestore import ModuleStoreEnum from xmodule.modulestore.django import modulestore +from xmodule.modulestore.exceptions import ItemNotFoundError from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase, TEST_DATA_SPLIT_MODULESTORE from xmodule.modulestore.tests.factories import ItemFactory, LibraryFactory, check_mongo_calls, CourseFactory from xmodule.x_module import STUDIO_VIEW, STUDENT_VIEW @@ -41,7 +45,9 @@ from xblock_django.user_service import DjangoXBlockUserService from opaque_keys.edx.keys import UsageKey, CourseKey from opaque_keys.edx.locations import Location -from xmodule.partitions.partitions import Group, UserPartition +from xmodule.partitions.partitions import ( + Group, UserPartition, ENROLLMENT_TRACK_PARTITION_ID, MINIMUM_STATIC_PARTITION_ID +) class AsideTest(XBlockAside): @@ -339,15 +345,17 @@ def test_invalid_paging(self, page_number, page_size): ) def test_get_user_partitions_and_groups(self): + # Note about UserPartition and UserPartition Group IDs: these must not conflict with IDs used + # by dynamic user partitions. self.course.user_partitions = [ UserPartition( - id=0, - name="Verification user partition", - scheme=UserPartition.get_scheme("verification"), - description="Verification user partition", + id=MINIMUM_STATIC_PARTITION_ID, + name="Random user partition", + scheme=UserPartition.get_scheme("random"), + description="Random user partition", groups=[ - Group(id=0, name="Group A"), - Group(id=1, name="Group B"), + Group(id=MINIMUM_STATIC_PARTITION_ID + 1, name="Group A"), # See note above. + Group(id=MINIMUM_STATIC_PARTITION_ID + 2, name="Group B"), # See note above. ], ), ] @@ -363,18 +371,31 @@ def test_get_user_partitions_and_groups(self): result = json.loads(resp.content) self.assertEqual(result["user_partitions"], [ { - "id": 0, - "name": "Verification user partition", - "scheme": "verification", + "id": ENROLLMENT_TRACK_PARTITION_ID, + "name": "Enrollment Track Groups", + "scheme": "enrollment_track", + "groups": [ + { + "id": settings.COURSE_ENROLLMENT_MODES["audit"], + "name": "Audit", + "selected": False, + "deleted": False, + } + ] + }, + { + "id": MINIMUM_STATIC_PARTITION_ID, + "name": "Random user partition", + "scheme": "random", "groups": [ { - "id": 0, + "id": MINIMUM_STATIC_PARTITION_ID + 1, "name": "Group A", "selected": False, "deleted": False, }, { - "id": 1, + "id": MINIMUM_STATIC_PARTITION_ID + 2, "name": "Group B", "selected": False, "deleted": False, @@ -384,6 +405,59 @@ def test_get_user_partitions_and_groups(self): ]) self.assertEqual(result["group_access"], {}) + @ddt.data('ancestorInfo', '') + def test_ancestor_info(self, field_type): + """ + Test that we get correct ancestor info. + + Arguments: + field_type (string): If field_type=ancestorInfo, fetch ancestor info of the XBlock otherwise not. + """ + + # Create a parent chapter + chap1 = self.create_xblock(parent_usage_key=self.course.location, display_name='chapter1', category='chapter') + chapter_usage_key = self.response_usage_key(chap1) + + # create a sequential + seq1 = self.create_xblock(parent_usage_key=chapter_usage_key, display_name='seq1', category='sequential') + seq_usage_key = self.response_usage_key(seq1) + + # create a vertical + vert1 = self.create_xblock(parent_usage_key=seq_usage_key, display_name='vertical1', category='vertical') + vert_usage_key = self.response_usage_key(vert1) + + # create problem and an html component + problem1 = self.create_xblock(parent_usage_key=vert_usage_key, display_name='problem1', category='problem') + problem_usage_key = self.response_usage_key(problem1) + + def assert_xblock_info(xblock, xblock_info): + """ + Assert we have correct xblock info. + + Arguments: + xblock (XBlock): An XBlock item. + xblock_info (dict): A dict containing xblock information. + """ + self.assertEqual(unicode(xblock.location), xblock_info['id']) + self.assertEqual(xblock.display_name, xblock_info['display_name']) + self.assertEqual(xblock.category, xblock_info['category']) + + for usage_key in (problem_usage_key, vert_usage_key, seq_usage_key, chapter_usage_key): + xblock = self.get_item_from_modulestore(usage_key) + url = reverse_usage_url('xblock_handler', usage_key) + '?fields={field_type}'.format(field_type=field_type) + response = self.client.get(url) + self.assertEqual(response.status_code, 200) + response = json.loads(response.content) + if field_type == 'ancestorInfo': + self.assertIn('ancestors', response) + for ancestor_info in response['ancestors']: + parent_xblock = xblock.get_parent() + assert_xblock_info(parent_xblock, ancestor_info) + xblock = parent_xblock + else: + self.assertNotIn('ancestors', response) + self.assertEqual(_get_module_info(xblock), response) + @ddt.ddt class DeleteItem(ItemTest): @@ -680,6 +754,500 @@ def verify_name(source_usage_key, parent_usage_key, expected_name, display_name= verify_name(self.seq_usage_key, self.chapter_usage_key, "customized name", display_name="customized name") +@ddt.ddt +class TestMoveItem(ItemTest): + """ + Tests for move item. + """ + def setUp(self): + """ + Creates the test course structure to build course outline tree. + """ + super(TestMoveItem, self).setUp() + self.setup_course() + + def setup_course(self, default_store=None): + """ + Helper method to create the course. + """ + if not default_store: + default_store = self.store.default_modulestore.get_modulestore_type() + + self.course = CourseFactory.create(default_store=default_store) + + # Create group configurations + self.course.user_partitions = [ + UserPartition(0, 'first_partition', 'Test Partition', [Group("0", 'alpha'), Group("1", 'beta')]) + ] + self.store.update_item(self.course, self.user.id) + + # Create a parent chapter + chap1 = self.create_xblock(parent_usage_key=self.course.location, display_name='chapter1', category='chapter') + self.chapter_usage_key = self.response_usage_key(chap1) + + chap2 = self.create_xblock(parent_usage_key=self.course.location, display_name='chapter2', category='chapter') + self.chapter2_usage_key = self.response_usage_key(chap2) + + # Create a sequential + seq1 = self.create_xblock(parent_usage_key=self.chapter_usage_key, display_name='seq1', category='sequential') + self.seq_usage_key = self.response_usage_key(seq1) + + seq2 = self.create_xblock(parent_usage_key=self.chapter_usage_key, display_name='seq2', category='sequential') + self.seq2_usage_key = self.response_usage_key(seq2) + + # Create a vertical + vert1 = self.create_xblock(parent_usage_key=self.seq_usage_key, display_name='vertical1', category='vertical') + self.vert_usage_key = self.response_usage_key(vert1) + + vert2 = self.create_xblock(parent_usage_key=self.seq_usage_key, display_name='vertical2', category='vertical') + self.vert2_usage_key = self.response_usage_key(vert2) + + # Create problem and an html component + problem1 = self.create_xblock(parent_usage_key=self.vert_usage_key, display_name='problem1', category='problem') + self.problem_usage_key = self.response_usage_key(problem1) + + html1 = self.create_xblock(parent_usage_key=self.vert_usage_key, display_name='html1', category='html') + self.html_usage_key = self.response_usage_key(html1) + + # Create a content experiment + resp = self.create_xblock(category='split_test', parent_usage_key=self.vert_usage_key) + self.split_test_usage_key = self.response_usage_key(resp) + + def setup_and_verify_content_experiment(self, partition_id): + """ + Helper method to set up group configurations to content experiment. + + Arguments: + partition_id (int): User partition id. + """ + split_test = self.get_item_from_modulestore(self.split_test_usage_key, verify_is_draft=True) + + # Initially, no user_partition_id is set, and the split_test has no children. + self.assertEqual(split_test.user_partition_id, -1) + self.assertEqual(len(split_test.children), 0) + + # Set group configuration + self.client.ajax_post( + reverse_usage_url("xblock_handler", self.split_test_usage_key), + data={'metadata': {'user_partition_id': str(partition_id)}} + ) + split_test = self.get_item_from_modulestore(self.split_test_usage_key, verify_is_draft=True) + self.assertEqual(split_test.user_partition_id, partition_id) + self.assertEqual(len(split_test.children), len(self.course.user_partitions[partition_id].groups)) + return split_test + + def _move_component(self, source_usage_key, target_usage_key, target_index=None): + """ + Helper method to send move request and returns the response. + + Arguments: + source_usage_key (BlockUsageLocator): Locator of source item. + target_usage_key (BlockUsageLocator): Locator of target parent. + target_index (int): If provided, insert source item at the provided index location in target_usage_key item. + + Returns: + resp (JsonResponse): Response after the move operation is complete. + """ + data = { + 'move_source_locator': unicode(source_usage_key), + 'parent_locator': unicode(target_usage_key) + } + if target_index is not None: + data['target_index'] = target_index + + return self.client.patch( + reverse('contentstore.views.xblock_handler'), + json.dumps(data), + content_type='application/json' + ) + + def assert_move_item(self, source_usage_key, target_usage_key, target_index=None): + """ + Assert move component. + + Arguments: + source_usage_key (BlockUsageLocator): Locator of source item. + target_usage_key (BlockUsageLocator): Locator of target parent. + target_index (int): If provided, insert source item at the provided index location in target_usage_key item. + """ + parent_loc = self.store.get_parent_location(source_usage_key) + parent = self.get_item_from_modulestore(parent_loc) + source_index = _get_source_index(source_usage_key, parent) + expected_index = target_index if target_index is not None else source_index + response = self._move_component(source_usage_key, target_usage_key, target_index) + self.assertEqual(response.status_code, 200) + response = json.loads(response.content) + self.assertEqual(response['move_source_locator'], unicode(source_usage_key)) + self.assertEqual(response['parent_locator'], unicode(target_usage_key)) + self.assertEqual(response['source_index'], expected_index) + + # Verify parent referance has been changed now. + new_parent_loc = self.store.get_parent_location(source_usage_key) + source_item = self.get_item_from_modulestore(source_usage_key) + self.assertEqual(source_item.parent, new_parent_loc) + self.assertEqual(new_parent_loc, target_usage_key) + self.assertNotEqual(parent_loc, new_parent_loc) + + # Assert item is present in children list of target parent and not source parent + target_parent = self.get_item_from_modulestore(target_usage_key) + source_parent = self.get_item_from_modulestore(parent_loc) + self.assertIn(source_usage_key, target_parent.children) + self.assertNotIn(source_usage_key, source_parent.children) + + @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) + def test_move_component(self, store_type): + """ + Test move component with different xblock types. + + Arguments: + store_type (ModuleStoreEnum.Type): Type of modulestore to create test course in. + """ + self.setup_course(default_store=store_type) + for source_usage_key, target_usage_key in [ + (self.html_usage_key, self.vert2_usage_key), + (self.vert_usage_key, self.seq2_usage_key), + (self.seq_usage_key, self.chapter2_usage_key) + ]: + self.assert_move_item(source_usage_key, target_usage_key) + + def test_move_source_index(self): + """ + Test moving an item to a particular index. + """ + parent = self.get_item_from_modulestore(self.vert_usage_key) + children = parent.get_children() + self.assertEqual(len(children), 3) + + # Create a component within vert2. + resp = self.create_xblock(parent_usage_key=self.vert2_usage_key, display_name='html2', category='html') + html2_usage_key = self.response_usage_key(resp) + + # Move html2_usage_key inside vert_usage_key at second position. + self.assert_move_item(html2_usage_key, self.vert_usage_key, 1) + parent = self.get_item_from_modulestore(self.vert_usage_key) + children = parent.get_children() + self.assertEqual(len(children), 4) + self.assertEqual(children[1].location, html2_usage_key) + + def test_move_undo(self): + """ + Test move a component and move it back (undo). + """ + # Get the initial index of the component + parent = self.get_item_from_modulestore(self.vert_usage_key) + original_index = _get_source_index(self.html_usage_key, parent) + + # Move component and verify that response contains initial index + response = self._move_component(self.html_usage_key, self.vert2_usage_key) + response = json.loads(response.content) + self.assertEquals(original_index, response['source_index']) + + # Verify that new parent has the moved component at the last index. + parent = self.get_item_from_modulestore(self.vert2_usage_key) + self.assertEqual(self.html_usage_key, parent.children[-1]) + + # Verify original and new index is different now. + source_index = _get_source_index(self.html_usage_key, parent) + self.assertNotEquals(original_index, source_index) + + # Undo Move to the original index, use the source index fetched from the response. + response = self._move_component(self.html_usage_key, self.vert_usage_key, response['source_index']) + response = json.loads(response.content) + self.assertEquals(original_index, response['source_index']) + + def test_move_large_target_index(self): + """ + Test moving an item at a large index would generate an error message. + """ + parent = self.get_item_from_modulestore(self.vert2_usage_key) + parent_children_length = len(parent.children) + response = self._move_component(self.html_usage_key, self.vert2_usage_key, parent_children_length + 10) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + + expected_error = 'You can not move {usage_key} at an invalid index ({target_index}).'.format( + usage_key=self.html_usage_key, + target_index=parent_children_length + 10 + ) + self.assertEqual(expected_error, response['error']) + new_parent_loc = self.store.get_parent_location(self.html_usage_key) + self.assertEqual(new_parent_loc, self.vert_usage_key) + + def test_invalid_move(self): + """ + Test invalid move. + """ + parent_loc = self.store.get_parent_location(self.html_usage_key) + response = self._move_component(self.html_usage_key, self.seq_usage_key) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + + expected_error = 'You can not move {source_type} into {target_type}.'.format( + source_type=self.html_usage_key.block_type, + target_type=self.seq_usage_key.block_type + ) + self.assertEqual(expected_error, response['error']) + new_parent_loc = self.store.get_parent_location(self.html_usage_key) + self.assertEqual(new_parent_loc, parent_loc) + + def test_move_current_parent(self): + """ + Test that a component can not be moved to it's current parent. + """ + parent_loc = self.store.get_parent_location(self.html_usage_key) + self.assertEqual(parent_loc, self.vert_usage_key) + response = self._move_component(self.html_usage_key, self.vert_usage_key) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + + self.assertEqual(response['error'], 'Item is already present in target location.') + self.assertEqual(self.store.get_parent_location(self.html_usage_key), parent_loc) + + def test_can_not_move_into_itself(self): + """ + Test that a component can not be moved to itself. + """ + library_content = self.create_xblock( + parent_usage_key=self.vert_usage_key, display_name='library content block', category='library_content' + ) + library_content_usage_key = self.response_usage_key(library_content) + parent_loc = self.store.get_parent_location(library_content_usage_key) + self.assertEqual(parent_loc, self.vert_usage_key) + response = self._move_component(library_content_usage_key, library_content_usage_key) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + + self.assertEqual(response['error'], 'You can not move an item into itself.') + self.assertEqual(self.store.get_parent_location(self.html_usage_key), parent_loc) + + def test_move_library_content(self): + """ + Test that library content can be moved to any other valid location. + """ + library_content = self.create_xblock( + parent_usage_key=self.vert_usage_key, display_name='library content block', category='library_content' + ) + library_content_usage_key = self.response_usage_key(library_content) + parent_loc = self.store.get_parent_location(library_content_usage_key) + self.assertEqual(parent_loc, self.vert_usage_key) + self.assert_move_item(library_content_usage_key, self.vert2_usage_key) + + def test_move_into_library_content(self): + """ + Test that a component can be moved into library content. + """ + library_content = self.create_xblock( + parent_usage_key=self.vert_usage_key, display_name='library content block', category='library_content' + ) + library_content_usage_key = self.response_usage_key(library_content) + self.assert_move_item(self.html_usage_key, library_content_usage_key) + + def test_move_content_experiment(self): + """ + Test that a content experiment can be moved. + """ + self.setup_and_verify_content_experiment(0) + + # Move content experiment + self.assert_move_item(self.split_test_usage_key, self.vert2_usage_key) + + def test_move_content_experiment_components(self): + """ + Test that component inside content experiment can be moved to any other valid location. + """ + split_test = self.setup_and_verify_content_experiment(0) + + # Add html component to Group A. + html1 = self.create_xblock( + parent_usage_key=split_test.children[0], display_name='html1', category='html' + ) + html_usage_key = self.response_usage_key(html1) + + # Move content experiment + self.assert_move_item(html_usage_key, self.vert2_usage_key) + + def test_move_into_content_experiment_groups(self): + """ + Test that a component can be moved to content experiment groups. + """ + split_test = self.setup_and_verify_content_experiment(0) + self.assert_move_item(self.html_usage_key, split_test.children[0]) + + def test_can_not_move_into_content_experiment_level(self): + """ + Test that a component can not be moved directly to content experiment level. + """ + self.setup_and_verify_content_experiment(0) + response = self._move_component(self.html_usage_key, self.split_test_usage_key) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + + self.assertEqual(response['error'], 'You can not move an item directly into content experiment.') + self.assertEqual(self.store.get_parent_location(self.html_usage_key), self.vert_usage_key) + + def test_can_not_move_content_experiment_into_its_children(self): + """ + Test that a content experiment can not be moved inside any of it's children. + """ + split_test = self.setup_and_verify_content_experiment(0) + + # Try to move content experiment inside it's child groups. + for child_vert_usage_key in split_test.children: + response = self._move_component(self.split_test_usage_key, child_vert_usage_key) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + + self.assertEqual(response['error'], 'You can not move an item into it\'s child.') + self.assertEqual(self.store.get_parent_location(self.split_test_usage_key), self.vert_usage_key) + + # Create content experiment inside group A and set it's group configuration. + resp = self.create_xblock(category='split_test', parent_usage_key=split_test.children[0]) + child_split_test_usage_key = self.response_usage_key(resp) + self.client.ajax_post( + reverse_usage_url("xblock_handler", child_split_test_usage_key), + data={'metadata': {'user_partition_id': str(0)}} + ) + child_split_test = self.get_item_from_modulestore(self.split_test_usage_key, verify_is_draft=True) + + # Try to move content experiment further down the level to a child group A nested inside main group A. + response = self._move_component(self.split_test_usage_key, child_split_test.children[0]) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + + self.assertEqual(response['error'], 'You can not move an item into it\'s child.') + self.assertEqual(self.store.get_parent_location(self.split_test_usage_key), self.vert_usage_key) + + def test_move_invalid_source_index(self): + """ + Test moving an item to an invalid index. + """ + target_index = 'test_index' + parent_loc = self.store.get_parent_location(self.html_usage_key) + response = self._move_component(self.html_usage_key, self.vert2_usage_key, target_index) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + + error = 'You must provide target_index ({target_index}) as an integer.'.format(target_index=target_index) + self.assertEqual(response['error'], error) + new_parent_loc = self.store.get_parent_location(self.html_usage_key) + self.assertEqual(new_parent_loc, parent_loc) + + def test_move_no_target_locator(self): + """ + Test move an item without specifying the target location. + """ + data = {'move_source_locator': unicode(self.html_usage_key)} + with self.assertRaises(InvalidKeyError): + self.client.patch( + reverse('contentstore.views.xblock_handler'), + json.dumps(data), + content_type='application/json' + ) + + def test_no_move_source_locator(self): + """ + Test patch request without providing a move source locator. + """ + response = self.client.patch( + reverse('contentstore.views.xblock_handler') + ) + self.assertEqual(response.status_code, 400) + response = json.loads(response.content) + self.assertEqual(response['error'], 'Patch request did not recognise any parameters to handle.') + + @patch('contentstore.views.item.log') + def test_move_logging(self, mock_logger): + """ + Test logging when an item is successfully moved. + + Arguments: + mock_logger (object): A mock logger object. + """ + insert_at = 0 + self.assert_move_item(self.html_usage_key, self.vert2_usage_key, insert_at) + mock_logger.info.assert_called_with( + 'MOVE: %s moved from %s to %s at %d index', + unicode(self.html_usage_key), + unicode(self.vert_usage_key), + unicode(self.vert2_usage_key), + insert_at + ) + + @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) + def test_move_and_discard_changes(self, store_type): + """ + Verifies that discard changes operation brings moved component back to source location and removes the component + from target location. + + Arguments: + store_type (ModuleStoreEnum.Type): Type of modulestore to create test course in. + """ + self.setup_course(default_store=store_type) + + old_parent_loc = self.store.get_parent_location(self.html_usage_key) + + # Check that old_parent_loc is not yet published. + self.assertFalse(self.store.has_item(old_parent_loc, revision=ModuleStoreEnum.RevisionOption.published_only)) + + # Publish old_parent_loc unit + self.client.ajax_post( + reverse_usage_url("xblock_handler", old_parent_loc), + data={'publish': 'make_public'} + ) + + # Check that old_parent_loc is now published. + self.assertTrue(self.store.has_item(old_parent_loc, revision=ModuleStoreEnum.RevisionOption.published_only)) + self.assertFalse(self.store.has_changes(self.store.get_item(old_parent_loc))) + + # Move component html_usage_key in vert2_usage_key + self.assert_move_item(self.html_usage_key, self.vert2_usage_key) + + # Check old_parent_loc becomes in draft mode now. + self.assertTrue(self.store.has_changes(self.store.get_item(old_parent_loc))) + + # Now discard changes in old_parent_loc + self.client.ajax_post( + reverse_usage_url("xblock_handler", old_parent_loc), + data={'publish': 'discard_changes'} + ) + + # Check that old_parent_loc now is reverted to publish. Changes discarded, html_usage_key moved back. + self.assertTrue(self.store.has_item(old_parent_loc, revision=ModuleStoreEnum.RevisionOption.published_only)) + self.assertFalse(self.store.has_changes(self.store.get_item(old_parent_loc))) + + # Now source item should be back in the old parent. + source_item = self.get_item_from_modulestore(self.html_usage_key) + self.assertEqual(source_item.parent, old_parent_loc) + self.assertEqual(self.store.get_parent_location(self.html_usage_key), source_item.parent) + + # Also, check that item is not present in target parent but in source parent + target_parent = self.get_item_from_modulestore(self.vert2_usage_key) + source_parent = self.get_item_from_modulestore(old_parent_loc) + self.assertIn(self.html_usage_key, source_parent.children) + self.assertNotIn(self.html_usage_key, target_parent.children) + + @ddt.data(ModuleStoreEnum.Type.mongo, ModuleStoreEnum.Type.split) + def test_move_item_not_found(self, store_type=ModuleStoreEnum.Type.mongo): + """ + Test that an item not found exception raised when an item is not found when getting the item. + + Arguments: + store_type (ModuleStoreEnum.Type): Type of modulestore to create test course in. + """ + self.setup_course(default_store=store_type) + + data = { + 'move_source_locator': unicode(self.usage_key.course_key.make_usage_key('html', 'html_test')), + 'parent_locator': unicode(self.vert2_usage_key) + } + with self.assertRaises(ItemNotFoundError): + self.client.patch( + reverse('contentstore.views.xblock_handler'), + json.dumps(data), + content_type='application/json' + ) + + class TestDuplicateItemWithAsides(ItemTest, DuplicateHelper): """ Test the duplicate method for blocks with asides. @@ -1239,15 +1807,30 @@ class TestEditSplitModule(ItemTest): def setUp(self): super(TestEditSplitModule, self).setUp() self.user = UserFactory() + + self.first_user_partition_group_1 = Group(unicode(MINIMUM_STATIC_PARTITION_ID + 1), 'alpha') + self.first_user_partition_group_2 = Group(unicode(MINIMUM_STATIC_PARTITION_ID + 2), 'beta') + self.first_user_partition = UserPartition( + MINIMUM_STATIC_PARTITION_ID, 'first_partition', 'First Partition', + [self.first_user_partition_group_1, self.first_user_partition_group_2] + ) + + # There is a test point below (test_create_groups) that purposefully wants the group IDs + # of the 2 partitions to overlap (which is not something that normally happens). + self.second_user_partition_group_1 = Group(unicode(MINIMUM_STATIC_PARTITION_ID + 1), 'Group 1') + self.second_user_partition_group_2 = Group(unicode(MINIMUM_STATIC_PARTITION_ID + 2), 'Group 2') + self.second_user_partition_group_3 = Group(unicode(MINIMUM_STATIC_PARTITION_ID + 3), 'Group 3') + self.second_user_partition = UserPartition( + MINIMUM_STATIC_PARTITION_ID + 10, 'second_partition', 'Second Partition', + [ + self.second_user_partition_group_1, + self.second_user_partition_group_2, + self.second_user_partition_group_3 + ] + ) self.course.user_partitions = [ - UserPartition( - 0, 'first_partition', 'First Partition', - [Group("0", 'alpha'), Group("1", 'beta')] - ), - UserPartition( - 1, 'second_partition', 'Second Partition', - [Group("0", 'Group 0'), Group("1", 'Group 1'), Group("2", 'Group 2')] - ) + self.first_user_partition, + self.second_user_partition ] self.store.update_item(self.course, self.user.id) root_usage_key = self._create_vertical() @@ -1295,8 +1878,8 @@ def test_create_groups(self): self.assertEqual(-1, split_test.user_partition_id) self.assertEqual(0, len(split_test.children)) - # Set the user_partition_id to 0. - split_test = self._update_partition_id(0) + # Set the user_partition_id to match the first user_partition. + split_test = self._update_partition_id(self.first_user_partition.id) # Verify that child verticals have been set to match the groups self.assertEqual(2, len(split_test.children)) @@ -1304,13 +1887,38 @@ def test_create_groups(self): vertical_1 = self.get_item_from_modulestore(split_test.children[1], verify_is_draft=True) self.assertEqual("vertical", vertical_0.category) self.assertEqual("vertical", vertical_1.category) - self.assertEqual("Group ID 0", vertical_0.display_name) - self.assertEqual("Group ID 1", vertical_1.display_name) + self.assertEqual("Group ID " + unicode(MINIMUM_STATIC_PARTITION_ID + 1), vertical_0.display_name) + self.assertEqual("Group ID " + unicode(MINIMUM_STATIC_PARTITION_ID + 2), vertical_1.display_name) # Verify that the group_id_to_child mapping is correct. self.assertEqual(2, len(split_test.group_id_to_child)) - self.assertEqual(vertical_0.location, split_test.group_id_to_child['0']) - self.assertEqual(vertical_1.location, split_test.group_id_to_child['1']) + self.assertEqual(vertical_0.location, split_test.group_id_to_child[str(self.first_user_partition_group_1.id)]) + self.assertEqual(vertical_1.location, split_test.group_id_to_child[str(self.first_user_partition_group_2.id)]) + + def test_split_xblock_info_group_name(self): + """ + Test that concise outline for split test component gives display name as group name. + """ + split_test = self.get_item_from_modulestore(self.split_test_usage_key, verify_is_draft=True) + # Initially, no user_partition_id is set, and the split_test has no children. + self.assertEqual(split_test.user_partition_id, -1) + self.assertEqual(len(split_test.children), 0) + # Set the user_partition_id to match the first user_partition. + split_test = self._update_partition_id(self.first_user_partition.id) + # Verify that child verticals have been set to match the groups + self.assertEqual(len(split_test.children), 2) + + # Get xblock outline + xblock_info = create_xblock_info( + split_test, + is_concise=True, + include_child_info=True, + include_children_predicate=lambda xblock: xblock.has_children, + course=self.course, + user=self.request.user + ) + self.assertEqual(xblock_info['child_info']['children'][0]['display_name'], 'alpha') + self.assertEqual(xblock_info['child_info']['children'][1]['display_name'], 'beta') def test_change_user_partition_id(self): """ @@ -1318,13 +1926,13 @@ def test_change_user_partition_id(self): group configuration. """ # Set to first group configuration. - split_test = self._update_partition_id(0) + split_test = self._update_partition_id(self.first_user_partition.id) self.assertEqual(2, len(split_test.children)) initial_vertical_0_location = split_test.children[0] initial_vertical_1_location = split_test.children[1] # Set to second group configuration - split_test = self._update_partition_id(1) + split_test = self._update_partition_id(self.second_user_partition.id) # We don't remove existing children. self.assertEqual(5, len(split_test.children)) self.assertEqual(initial_vertical_0_location, split_test.children[0]) @@ -1335,9 +1943,9 @@ def test_change_user_partition_id(self): # Verify that the group_id_to child mapping is correct. self.assertEqual(3, len(split_test.group_id_to_child)) - self.assertEqual(vertical_0.location, split_test.group_id_to_child['0']) - self.assertEqual(vertical_1.location, split_test.group_id_to_child['1']) - self.assertEqual(vertical_2.location, split_test.group_id_to_child['2']) + self.assertEqual(vertical_0.location, split_test.group_id_to_child[str(self.second_user_partition_group_1.id)]) + self.assertEqual(vertical_1.location, split_test.group_id_to_child[str(self.second_user_partition_group_2.id)]) + self.assertEqual(vertical_2.location, split_test.group_id_to_child[str(self.second_user_partition_group_3.id)]) self.assertNotEqual(initial_vertical_0_location, vertical_0.location) self.assertNotEqual(initial_vertical_1_location, vertical_1.location) @@ -1346,12 +1954,12 @@ def test_change_same_user_partition_id(self): Test that nothing happens when the user_partition_id is set to the same value twice. """ # Set to first group configuration. - split_test = self._update_partition_id(0) + split_test = self._update_partition_id(self.first_user_partition.id) self.assertEqual(2, len(split_test.children)) initial_group_id_to_child = split_test.group_id_to_child # Set again to first group configuration. - split_test = self._update_partition_id(0) + split_test = self._update_partition_id(self.first_user_partition.id) self.assertEqual(2, len(split_test.children)) self.assertEqual(initial_group_id_to_child, split_test.group_id_to_child) @@ -1362,7 +1970,7 @@ def test_change_non_existent_user_partition_id(self): The user_partition_id will be updated, but children and group_id_to_child map will not change. """ # Set to first group configuration. - split_test = self._update_partition_id(0) + split_test = self._update_partition_id(self.first_user_partition.id) self.assertEqual(2, len(split_test.children)) initial_group_id_to_child = split_test.group_id_to_child @@ -1379,13 +1987,14 @@ def test_add_groups(self): TODO: move tests that can go over to common after the mixed modulestore work is done. # pylint: disable=fixme """ # Set to first group configuration. - split_test = self._update_partition_id(0) + split_test = self._update_partition_id(self.first_user_partition.id) # Add a group to the first group configuration. + new_group_id = "1002" split_test.user_partitions = [ UserPartition( - 0, 'first_partition', 'First Partition', - [Group("0", 'alpha'), Group("1", 'beta'), Group("2", 'pie')] + self.first_user_partition.id, 'first_partition', 'First Partition', + [self.first_user_partition_group_1, self.first_user_partition_group_2, Group(new_group_id, 'pie')] ) ] self.store.update_item(split_test, self.user.id) @@ -1406,7 +2015,7 @@ def test_add_groups(self): split_test = self._assert_children(3) self.assertNotEqual(group_id_to_child, split_test.group_id_to_child) group_id_to_child = split_test.group_id_to_child - self.assertEqual(split_test.children[2], group_id_to_child["2"]) + self.assertEqual(split_test.children[2], group_id_to_child[new_group_id]) # Call add_missing_groups again -- it should be a no-op. split_test.add_missing_groups(self.request) @@ -1649,7 +2258,7 @@ def get_xblock_problem(label): def verify_openassessment_present(support_level): """ Helper method to verify that openassessment template is present """ - openassessment = get_xblock_problem('Peer Assessment') + openassessment = get_xblock_problem('Open Response Assessment') self.assertIsNotNone(openassessment) self.assertEqual(openassessment.get('category'), 'openassessment') self.assertEqual(openassessment.get('support_level'), support_level) diff --git a/cms/djangoapps/contentstore/views/tests/test_library.py b/cms/djangoapps/contentstore/views/tests/test_library.py index 2a704c002d17..102d0778afe9 100644 --- a/cms/djangoapps/contentstore/views/tests/test_library.py +++ b/cms/djangoapps/contentstore/views/tests/test_library.py @@ -3,15 +3,19 @@ More important high-level tests are in contentstore/tests/test_libraries.py """ -from contentstore.tests.utils import AjaxEnabledTestClient, parse_json -from contentstore.utils import reverse_course_url, reverse_library_url -from contentstore.views.component import get_component_templates -from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import LibraryFactory +import ddt +import mock +from django.conf import settings from mock import patch from opaque_keys.edx.locator import CourseKey, LibraryLocator -import ddt + +from contentstore.tests.utils import AjaxEnabledTestClient, CourseTestCase, parse_json +from contentstore.utils import reverse_course_url, reverse_library_url +from contentstore.views.component import get_component_templates +from contentstore.views.library import get_library_creator_status +from course_creators.views import add_user_with_status_granted as grant_course_creator_status from student.roles import LibraryUserRole +from xmodule.modulestore.tests.factories import LibraryFactory LIBRARY_REST_URL = '/library/' # URL for GET/POST requests involving libraries @@ -24,7 +28,7 @@ def make_url_for_lib(key): @ddt.ddt -class UnitTestLibraries(ModuleStoreTestCase): +class UnitTestLibraries(CourseTestCase): """ Unit tests for library views """ @@ -38,6 +42,27 @@ def setUp(self): ###################################################### # Tests for /library/ - list and create libraries: + @mock.patch("contentstore.views.library.LIBRARIES_ENABLED", False) + def test_library_creator_status_libraries_not_enabled(self): + _, nostaff_user = self.create_non_staff_authed_user_client() + self.assertEqual(get_library_creator_status(nostaff_user), False) + + @mock.patch("contentstore.views.library.LIBRARIES_ENABLED", True) + def test_library_creator_status_with_is_staff_user(self): + self.assertEqual(get_library_creator_status(self.user), True) + + @mock.patch("contentstore.views.library.LIBRARIES_ENABLED", True) + def test_library_creator_status_with_course_creator_role(self): + _, nostaff_user = self.create_non_staff_authed_user_client() + with mock.patch.dict('django.conf.settings.FEATURES', {"ENABLE_CREATOR_GROUP": True}): + grant_course_creator_status(self.user, nostaff_user) + self.assertEqual(get_library_creator_status(nostaff_user), True) + + @mock.patch("contentstore.views.library.LIBRARIES_ENABLED", True) + def test_library_creator_status_with_no_course_creator_role(self): + _, nostaff_user = self.create_non_staff_authed_user_client() + self.assertEqual(get_library_creator_status(nostaff_user), True) + @patch("contentstore.views.library.LIBRARIES_ENABLED", False) def test_with_libraries_disabled(self): """ @@ -87,18 +112,45 @@ def test_create_library(self): @patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': True}) def test_lib_create_permission(self): """ - Users who are not given course creator roles should still be able to - create libraries. + Users who are given course creator roles should be able to create libraries. """ self.client.logout() ns_user, password = self.create_non_staff_user() self.client.login(username=ns_user.username, password=password) + grant_course_creator_status(self.user, ns_user) + response = self.client.ajax_post(LIBRARY_REST_URL, { + 'org': 'org', 'library': 'lib', 'display_name': "New Library", + }) + self.assertEqual(response.status_code, 200) + @patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': False}) + def test_lib_create_permission_no_course_creator_role_and_no_course_creator_group(self): + """ + Users who are not given course creator roles should still be able to create libraries + if ENABLE_CREATOR_GROUP is not enabled. + """ + self.client.logout() + ns_user, password = self.create_non_staff_user() + self.client.login(username=ns_user.username, password=password) response = self.client.ajax_post(LIBRARY_REST_URL, { 'org': 'org', 'library': 'lib', 'display_name': "New Library", }) self.assertEqual(response.status_code, 200) + @patch.dict('django.conf.settings.FEATURES', {'ENABLE_CREATOR_GROUP': True}) + def test_lib_create_permission_no_course_creator_role_and_course_creator_group(self): + """ + Users who are not given course creator roles should not be able to create libraries + if ENABLE_CREATOR_GROUP is enabled. + """ + self.client.logout() + ns_user, password = self.create_non_staff_user() + self.client.login(username=ns_user.username, password=password) + response = self.client.ajax_post(LIBRARY_REST_URL, { + 'org': 'org', 'library': 'lib', 'display_name': "New Library", + }) + self.assertEqual(response.status_code, 403) + @ddt.data( {}, {'org': 'org'}, @@ -160,7 +212,7 @@ def test_get_lib_edit_html(self): response = self.client.get(make_url_for_lib(lib.location.library_key)) self.assertEqual(response.status_code, 200) self.assertIn(" + + + + """.format(youtube_id=youtube_id) + ) + modulestore().update_item(self.item, self.user.id) + self.assert_current_subs(expected_subs='') + + # Save new subs in the content store. + subs = { + 'start': [100, 200, 240], + 'end': [200, 240, 380], + 'text': [ + 'subs #1', + 'subs #2', + 'subs #3' + ] + } + self.save_subs_to_store(subs, youtube_id) + + # Now, make request to /transcripts/save endpoint with new subs. + data = { + 'locator': unicode(self.video_usage_key), + 'metadata': { + 'sub': youtube_id + } + } + resp = self.client.get(reverse('save_transcripts'), {'data': json.dumps(data)}) + self.assertEqual(resp.status_code, 200) + self.assertEqual(json.loads(resp.content), {"status": "Success"}) + + # Now check item.sub, it should be same as youtube id because /transcripts/save prioritize + # youtube subs over html5 ones. + self.assert_current_subs(expected_subs=youtube_id) diff --git a/cms/djangoapps/contentstore/views/tests/test_user.py b/cms/djangoapps/contentstore/views/tests/test_user.py index 725858e44de5..a2bd27b51404 100644 --- a/cms/djangoapps/contentstore/views/tests/test_user.py +++ b/cms/djangoapps/contentstore/views/tests/test_user.py @@ -3,12 +3,13 @@ """ import json +from django.contrib.auth.models import User + from contentstore.tests.utils import CourseTestCase from contentstore.utils import reverse_course_url -from django.contrib.auth.models import User -from student.models import CourseEnrollment -from student.roles import CourseStaffRole, CourseInstructorRole from student import auth +from student.models import CourseEnrollment +from student.roles import CourseInstructorRole, CourseStaffRole class UsersTestCase(CourseTestCase): diff --git a/cms/djangoapps/contentstore/views/tests/test_videos.py b/cms/djangoapps/contentstore/views/tests/test_videos.py index f91b188d320f..6731bf68158a 100644 --- a/cms/djangoapps/contentstore/views/tests/test_videos.py +++ b/cms/djangoapps/contentstore/views/tests/test_videos.py @@ -2,25 +2,24 @@ """ Unit tests for video-related REST APIs. """ -from datetime import datetime import csv -import ddt import json -import dateutil.parser import re +from datetime import datetime from StringIO import StringIO -import pytz +import dateutil.parser +import ddt +import pytz from django.conf import settings from django.test.utils import override_settings -from mock import Mock, patch - from edxval.api import create_profile, create_video, get_video_info +from mock import Mock, patch from contentstore.models import VideoUploadConfig -from contentstore.views.videos import KEY_EXPIRATION_IN_SECONDS, StatusDisplayStrings, convert_video_status from contentstore.tests.utils import CourseTestCase from contentstore.utils import reverse_course_url +from contentstore.views.videos import KEY_EXPIRATION_IN_SECONDS, StatusDisplayStrings, convert_video_status from xmodule.modulestore.tests.factories import CourseFactory @@ -488,13 +487,13 @@ def assert_video_status(self, url, edx_video_id, status): # Test should fail if video not found self.assertEqual(True, False, 'Invalid edx_video_id') - def test_video_status_update_request(self): + @patch('contentstore.views.videos.LOGGER') + def test_video_status_update_request(self, mock_logger): """ Verifies that video status update request works as expected. """ url = self.get_url_for_course_key(self.course.id) edx_video_id = 'test1' - self.assert_video_status(url, edx_video_id, 'Uploading') response = self.client.post( @@ -506,6 +505,14 @@ def test_video_status_update_request(self): }]), content_type="application/json" ) + + mock_logger.info.assert_called_with( + 'VIDEOS: Video status update with id [%s], status [%s] and message [%s]', + edx_video_id, + 'upload_failed', + 'server down' + ) + self.assertEqual(response.status_code, 204) self.assert_video_status(url, edx_video_id, 'Failed') diff --git a/cms/djangoapps/contentstore/views/tests/utils.py b/cms/djangoapps/contentstore/views/tests/utils.py index 094a789214be..6c2580fd1a4c 100644 --- a/cms/djangoapps/contentstore/views/tests/utils.py +++ b/cms/djangoapps/contentstore/views/tests/utils.py @@ -41,33 +41,48 @@ def get_preview_html(self, xblock, view_name): resp_content = json.loads(resp.content) return resp_content['html'] - def validate_preview_html(self, xblock, view_name, can_add=True): + def validate_preview_html(self, xblock, view_name, can_add=True, can_reorder=True, can_move=True, + can_edit=True, can_duplicate=True, can_delete=True): """ Verify that the specified xblock's preview has the expected HTML elements. """ html = self.get_preview_html(xblock, view_name) - self.validate_html_for_add_buttons(html, can_add) + self.validate_html_for_action_button( + html, + '
', + can_add + ) + self.validate_html_for_action_button( + html, + '', + can_reorder + ) + self.validate_html_for_action_button( + html, + '
  • - + +
    • +
  • +
  • - +
    • diff --git a/cms/templates/container.html b/cms/templates/container.html index 43d8b78644d6..852fac1697e8 100644 --- a/cms/templates/container.html +++ b/cms/templates/container.html @@ -43,7 +43,8 @@ "${action | n, js_escaped_string}", { isUnitPage: ${is_unit_page | n, dump_js_escaped_json}, - canEdit: true + canEdit: true, + outlineURL: "${outline_url | n, js_escaped_string}" } ); }); @@ -144,7 +145,7 @@

    ${_("Unit Location")}

    ${_("Location ID")}

    ${unit.location.name} - Tip: ${_("Use this ID when you create links to this unit from other course content. You enter the ID in the URL field.")} + Tip: ${_('To create a link to this unit from an HTML component in this course, enter "/jump_to_id/" as the URL value.')}

    diff --git a/cms/templates/course-create-rerun.html b/cms/templates/course-create-rerun.html index 0be39608d123..a438af889513 100644 --- a/cms/templates/course-create-rerun.html +++ b/cms/templates/course-create-rerun.html @@ -91,12 +91,11 @@

  • -
    - - +
    + + - ${_("The unique number that identifies the new course within the organization. (This number is often the same as the original course number.)")} - ${_("Note: No spaces or special characters are allowed.")} + ${_("The unique number that identifies the new course within the organization. (This number will be the same as the original course number and cannot be changed.)")}
    diff --git a/cms/templates/course_info.html b/cms/templates/course_info.html index 0abb94e8229c..ebdec4b9c40b 100644 --- a/cms/templates/course_info.html +++ b/cms/templates/course_info.html @@ -63,7 +63,11 @@

    ${_('Page Actions')}

    -
    +
      diff --git a/cms/templates/course_outline.html b/cms/templates/course_outline.html index 1ec0213690f2..c0b61219f5f1 100644 --- a/cms/templates/course_outline.html +++ b/cms/templates/course_outline.html @@ -62,7 +62,7 @@

      ${_("This course was created as a re-run. Some manual ## Since we're unsure what's causing this, it's difficult to test, and we don't actually use the feature, ## we've decided to simply remove it from the page. %if False: - %if deprecated_blocks_info.get('blocks') or deprecated_blocks_info.get('block_types_enabled'): + %if deprecated_blocks_info.get('blocks') or deprecated_blocks_info.get('deprecated_enabled_block_types'):
      ${_("Warning")} @@ -89,7 +89,7 @@

      ${_("This course uses features th

      %endif - % if deprecated_blocks_info.get('block_types_enabled'): + % if deprecated_blocks_info.get('deprecated_enabled_block_types'):

      ${Text(_("To avoid errors, {platform_name} strongly recommends that you remove unsupported features from the course advanced settings. To do this, go to the {link_start}Advanced Settings page{link_end}, locate the \"Advanced Module List\" setting, and then delete the following modules from the list.")).format( @@ -100,7 +100,7 @@

      ${_("This course uses features th

      % endif
      - -
      +
      <% course_locator = context_course.location %> diff --git a/cms/templates/emails/user_task_complete_email.txt b/cms/templates/emails/user_task_complete_email.txt new file mode 100644 index 000000000000..05b49cc8b825 --- /dev/null +++ b/cms/templates/emails/user_task_complete_email.txt @@ -0,0 +1,11 @@ +<%! from django.utils.translation import ugettext as _ %> + +% if detail_url: + +${_("Your {task_name} task has completed with the status '{task_status}'. Use this URL to view task details or download any files created: {detail_url}").format(task_name=task_name, task_status=task_status, detail_url=detail_url)} + +% else: + +${_("Your {task_name} task has completed with the status '{task_status}'. Sign in to view the details of your task or download any files created.").format(task_name=task_name, task_status=task_status)} + +% endif diff --git a/cms/templates/emails/user_task_complete_email_subject.txt b/cms/templates/emails/user_task_complete_email_subject.txt new file mode 100644 index 000000000000..0876382638a8 --- /dev/null +++ b/cms/templates/emails/user_task_complete_email_subject.txt @@ -0,0 +1,2 @@ +<%! from django.utils.translation import ugettext as _ %> +${_("{platform_name} {studio_name}: Task Status Update").format(platform_name=settings.PLATFORM_NAME, studio_name=settings.STUDIO_NAME)} diff --git a/cms/templates/export.html b/cms/templates/export.html index f6982dabaf1b..1013fc3aab89 100644 --- a/cms/templates/export.html +++ b/cms/templates/export.html @@ -27,17 +27,13 @@ <%block name="bodyclass">is-signedin course tools view-export <%block name="requirejs"> -% if in_err: - var hasUnit = ${bool(unit) | n, dump_js_escaped_json}, - editUnitUrl = "${edit_unit_url | n, js_escaped_string}", - courselikeHomeUrl = "${courselike_home_url | n, js_escaped_string}", - is_library = ${library | n, dump_js_escaped_json} - errMsg = "${raw_err_msg | n, js_escaped_string}"; + var courselikeHomeUrl = "${courselike_home_url | n, js_escaped_string}", + is_library = ${library | n, dump_js_escaped_json}, + statusUrl = "${status_url | n, js_escaped_string}"; require(["js/factories/export"], function(ExportFactory) { - ExportFactory(hasUnit, editUnitUrl, courselikeHomeUrl, is_library, errMsg); + ExportFactory(courselikeHomeUrl, is_library, statusUrl); }); -%endif <%block name="content"> @@ -93,7 +89,7 @@

      + + + %if not library:
      diff --git a/cms/templates/group_configurations.html b/cms/templates/group_configurations.html index e883d2fed931..37bf79679582 100644 --- a/cms/templates/group_configurations.html +++ b/cms/templates/group_configurations.html @@ -1,6 +1,7 @@ <%page expression_filter="h"/> <%inherit file="base.html" /> <%def name="content_groups_help_token()"><% return "content_groups" %> +<%def name="enrollment_track_help_token()"><% return "enrollment_tracks" %> <%def name="experiment_group_configurations_help_token()"><% return "group_configurations" %> <%namespace name='static' file='static_content.html'/> <%! @@ -16,7 +17,7 @@ <%block name="bodyclass">is-signedin course view-group-configurations <%block name="header_extras"> -% for template_name in ["group-configuration-details", "group-configuration-editor", "group-edit", "content-group-editor", "content-group-details", "basic-modal", "modal-button", "list"]: +% for template_name in ["group-configuration-details", "group-configuration-editor", "group-edit", "content-group-editor", "partition-group-details", "basic-modal", "modal-button", "list"]: @@ -28,9 +29,10 @@ GroupConfigurationsFactory( ${should_show_experiment_groups | n, dump_js_escaped_json}, ${experiment_group_configurations | n, dump_js_escaped_json}, - ${content_group_configuration | n, dump_js_escaped_json}, + ${all_group_configurations | n, dump_js_escaped_json}, "${group_configuration_url | n, js_escaped_string}", - "${course_outline_url | n, js_escaped_string}" + "${course_outline_url | n, js_escaped_string}", + ${should_show_enrollment_track | n, dump_js_escaped_json} ); }); @@ -47,37 +49,52 @@

      +
      -
      -

      ${_("Content Groups")}

      -
      -

      ${_("Loading")}

      -
      -
      - % if should_show_experiment_groups: -
      -

      ${_("Experiment Group Configurations")}

      - % if experiment_group_configurations is None: -
      -

      - ${_("This module is disabled at the moment.")} -

      -
      - % else: -
      -

      ${_("Loading")}

      -
      - % endif -
      - % endif + % for config in all_group_configurations: +
      +

      ${config['name']}

      +
      +

      ${_("Loading")}

      +
      +
      + % endfor + + % if should_show_experiment_groups: +
      +

      ${_("Experiment Group Configurations")}

      + % if experiment_group_configurations is None: +
      +

      + ${_("This module is disabled at the moment.")} +

      +
      + % else: +
      +

      ${_("Loading")}

      +
      + % endif +
      + % endif
      +<%static:webpack entry="Import"> + Import('${import_status_url | n, js_escaped_string}', ${library | n, dump_js_escaped_json}); + -<%block name="requirejs"> - require(["js/factories/import"], function(ImportFactory) { - ImportFactory( - "${import_status_url | n, js_escaped_string}", - ${library | n, dump_js_escaped_json} - ); - }); - diff --git a/cms/templates/index.html b/cms/templates/index.html index 4fb412c7c157..f1b4375f18d4 100644 --- a/cms/templates/index.html +++ b/cms/templates/index.html @@ -33,16 +33,10 @@

      ${_("Page Actions")}

      % elif course_creator_status=='disallowed_for_this_site' and settings.FEATURES.get('STUDIO_REQUEST_EMAIL',''): ${_("Email staff to create course")} % endif - % if show_new_library_button: ${_("New Library")} % endif - - % if is_programs_enabled: - - ${_("New Program")} - % endif

      1. @@ -189,6 +183,30 @@

      ${_("Create a New Library")}

      % endif + %if optimization_enabled: +
      +

      ${_("Organization and Library Settings")}

      +
      +
      +
      + +
      +
      + +
      +
      +
      + +
      +
      +
      + %endif + %if allow_course_reruns and rerun_creator_status and len(in_process_course_actions) > 0:
      @@ -295,33 +313,14 @@

      ${course_info['display_name']}

      %endif - % if libraries_enabled or is_programs_enabled: + % if libraries_enabled: % endif - %if len(courses) > 0: + %if len(courses) > 0 or optimization_enabled:
        %for course_info in sorted(courses, key=lambda s: s['display_name'].lower() if s['display_name'] is not None else ''): @@ -528,52 +527,6 @@

        ${_('Create Your First Library')}

      %endif - % if is_programs_enabled: - % if len(programs) > 0: -
      - - -
      - - % else: -
      -
      - -
      -

      ${_("You haven't created any programs yet.")}

      -
      -

      ${_("Programs are groups of courses related to a common subject.")}

      -
      -
      - - - -
      -
      - % endif - % endif -