From 606b33c07ae9b14de968d03f6867cd2a0a11b9c4 Mon Sep 17 00:00:00 2001 From: Thibault Dethier Date: Fri, 20 Dec 2024 12:28:46 +0100 Subject: [PATCH] Added task instance_bulk_gps_push Refactored the bulk gps check in order to call it inside the task. Fixed typo in HasCreateOrgUnitPermission. --- hat/audit/models.py | 1 + iaso/api/instances.py | 80 ++-- iaso/api/org_units.py | 4 +- .../tasks/create/instance_bulk_gps_push.py | 33 ++ iaso/tasks/instance_bulk_gps_push.py | 76 ++++ iaso/tests/api/test_instances.py | 63 ++- .../tasks/test_instance_bulk_gps_push.py | 392 ++++++++++++++++++ iaso/urls.py | 2 + iaso/utils/models/common.py | 75 ++++ 9 files changed, 662 insertions(+), 64 deletions(-) create mode 100644 iaso/api/tasks/create/instance_bulk_gps_push.py create mode 100644 iaso/tasks/instance_bulk_gps_push.py create mode 100644 iaso/tests/tasks/test_instance_bulk_gps_push.py diff --git a/hat/audit/models.py b/hat/audit/models.py index f97f0e367b..c323d6e1a0 100644 --- a/hat/audit/models.py +++ b/hat/audit/models.py @@ -16,6 +16,7 @@ ORG_UNIT_API_BULK = "org_unit_api_bulk" GROUP_SET_API = "group_set_api" INSTANCE_API = "instance_api" +INSTANCE_API_BULK = "instance_api_bulk" FORM_API = "form_api" GPKG_IMPORT = "gpkg_import" CAMPAIGN_API = "campaign_api" diff --git a/iaso/api/instances.py b/iaso/api/instances.py index 8f87b5a8ea..22d36dbfcd 100644 --- a/iaso/api/instances.py +++ b/iaso/api/instances.py @@ -38,9 +38,10 @@ ) from iaso.utils import timestamp_to_datetime from iaso.utils.file_utils import get_file_type +from .org_units import HasCreateOrgUnitPermission from ..models.forms import CR_MODE_IF_REFERENCE_FORM -from ..utils.models.common import get_creator_name +from ..utils.models.common import get_creator_name, check_instance_bulk_gps_push from . import common from .comment import UserSerializerForComment from .common import ( @@ -91,7 +92,7 @@ def validate_period(self, value): class HasInstancePermission(permissions.BasePermission): def has_permission(self, request: Request, view): - if request.method == "POST": + if request.method == "POST": # to handle anonymous submissions sent by mobile return True return request.user.is_authenticated and ( @@ -112,6 +113,19 @@ def has_object_permission(self, request: Request, view, obj: Instance): return False +class HasInstanceBulkPermission(permissions.BasePermission): + """ + Designed for POST endpoints that are not designed to receive new submissions. + """ + def has_permission(self, request: Request, view): + return request.user.is_authenticated and ( + request.user.has_perm(permission.FORMS) + or request.user.has_perm(permission.SUBMISSIONS) + or request.user.has_perm(permission.REGISTRY_WRITE) + or request.user.has_perm(permission.REGISTRY_READ) + ) + + class InstanceFileSerializer(serializers.Serializer): id = serializers.IntegerField(read_only=True) instance_id = serializers.IntegerField() @@ -605,7 +619,11 @@ def bulkdelete(self, request): status=201, ) - @action(detail=False, methods=["GET"], permission_classes=[permissions.IsAuthenticated, HasInstancePermission]) + @action( + detail=False, + methods=["GET"], + permission_classes=[permissions.IsAuthenticated, HasInstanceBulkPermission, HasCreateOrgUnitPermission], + ) def check_bulk_gps_push(self, request): # first, let's parse all parameters received from the URL select_all, selected_ids, unselected_ids = self._parse_check_bulk_gps_push_parameters(request.GET) @@ -628,48 +646,15 @@ def check_bulk_gps_push(self, request): else: instances_query = instances_query.exclude(pk__in=unselected_ids) - overwrite_ids = [] - no_location_ids = [] - org_units_to_instances_dict = {} - set_org_units_ids = set() + success, errors, warnings = check_instance_bulk_gps_push(instances_query) - for instance in instances_query: - if not instance.location: - no_location_ids.append(instance.id) # there is nothing to push to the OrgUnit - continue + if not success: + errors["result"] = "errors" + return Response(errors, status=status.HTTP_400_BAD_REQUEST) - org_unit = instance.org_unit - if org_unit.id in org_units_to_instances_dict: - # we can't push this instance's location since there was another instance linked to this OrgUnit - org_units_to_instances_dict[org_unit.id].append(instance.id) - continue - else: - org_units_to_instances_dict[org_unit.id] = [instance.id] - - set_org_units_ids.add(org_unit.id) - if org_unit.location or org_unit.geom: - overwrite_ids.append(instance.id) # if the user proceeds, he will erase existing location - continue - - # Before returning, we need to check if we've had multiple hits on an OrgUnit - error_same_org_unit_ids = self._check_bulk_gps_repeated_org_units(org_units_to_instances_dict) - - if len(error_same_org_unit_ids): - return Response( - {"result": "error", "error_ids": error_same_org_unit_ids}, - status=status.HTTP_400_BAD_REQUEST, - ) - - if len(no_location_ids) or len(overwrite_ids): - dict_response = { - "result": "warnings", - } - if len(no_location_ids): - dict_response["warning_no_location"] = no_location_ids - if len(overwrite_ids): - dict_response["warning_overwrite"] = overwrite_ids - - return Response(dict_response, status=status.HTTP_200_OK) + if warnings: + warnings["result"] = "warnings" + return Response(warnings, status=status.HTTP_200_OK) return Response( { @@ -680,7 +665,7 @@ def check_bulk_gps_push(self, request): def _parse_check_bulk_gps_push_parameters(self, query_parameters): raw_select_all = query_parameters.get("select_all", True) - select_all = raw_select_all not in ["false", "False", "0"] + select_all = raw_select_all not in ["false", "False", "0", 0, False] raw_selected_ids = query_parameters.get("selected_ids", None) if raw_selected_ids: @@ -696,13 +681,6 @@ def _parse_check_bulk_gps_push_parameters(self, query_parameters): return select_all, selected_ids, unselected_ids - def _check_bulk_gps_repeated_org_units(self, org_units_to_instance_ids: Dict[int, List[int]]) -> List[int]: - error_instance_ids = [] - for _, instance_ids in org_units_to_instance_ids.items(): - if len(instance_ids) >= 2: - error_instance_ids.extend(instance_ids) - return error_instance_ids - QUERY = """ select DATE_TRUNC('month', COALESCE(iaso_instance.source_created_at, iaso_instance.created_at)) as month, (select name from iaso_form where id = iaso_instance.form_id) as form_name, diff --git a/iaso/api/org_units.py b/iaso/api/org_units.py index 45623ff7de..937b44b927 100644 --- a/iaso/api/org_units.py +++ b/iaso/api/org_units.py @@ -32,7 +32,7 @@ # noinspection PyMethodMayBeStatic -class HasCreateOrUnitPermission(permissions.BasePermission): +class HasCreateOrgUnitPermission(permissions.BasePermission): def has_permission(self, request, view): if not request.user.is_authenticated: return False @@ -614,7 +614,7 @@ def get_date(self, date: str) -> Union[datetime.date, None]: pass return None - @action(detail=False, methods=["POST"], permission_classes=[permissions.IsAuthenticated, HasCreateOrUnitPermission]) + @action(detail=False, methods=["POST"], permission_classes=[permissions.IsAuthenticated, HasCreateOrgUnitPermission]) def create_org_unit(self, request): """This endpoint is used by the React frontend""" errors = [] diff --git a/iaso/api/tasks/create/instance_bulk_gps_push.py b/iaso/api/tasks/create/instance_bulk_gps_push.py new file mode 100644 index 0000000000..3ba956269c --- /dev/null +++ b/iaso/api/tasks/create/instance_bulk_gps_push.py @@ -0,0 +1,33 @@ +from rest_framework import viewsets, permissions, status +from rest_framework.response import Response + +from iaso.api.instances import HasInstanceBulkPermission +from iaso.api.org_units import HasCreateOrgUnitPermission +from iaso.api.tasks import TaskSerializer +from iaso.tasks.instance_bulk_gps_push import instance_bulk_gps_push + + +class InstanceBulkGpsPushViewSet(viewsets.ViewSet): + """Bulk push gps location from Instances to their related OrgUnit. + + This task will override existing location on OrgUnits and might set `None` if the Instance doesn't have any location. + Calling this endpoint implies that the InstanceViewSet.check_bulk_gps_push() method has been called before and has returned no error. + """ + + permission_classes = [permissions.IsAuthenticated, HasInstanceBulkPermission, HasCreateOrgUnitPermission] + + def create(self, request): + raw_select_all = request.data.get("select_all", True) + select_all = raw_select_all not in [False, "false", "False", "0", 0] + selected_ids = request.data.get("selected_ids", []) + unselected_ids = request.data.get("unselected_ids", []) + + user = self.request.user + + task = instance_bulk_gps_push( + select_all=select_all, selected_ids=selected_ids, unselected_ids=unselected_ids, user=user + ) + return Response( + {"task": TaskSerializer(instance=task).data}, + status=status.HTTP_201_CREATED, + ) diff --git a/iaso/tasks/instance_bulk_gps_push.py b/iaso/tasks/instance_bulk_gps_push.py new file mode 100644 index 0000000000..53ac920d33 --- /dev/null +++ b/iaso/tasks/instance_bulk_gps_push.py @@ -0,0 +1,76 @@ +from copy import deepcopy +from logging import getLogger +from time import time +from typing import Optional, List + +from django.contrib.auth.models import User +from django.db import transaction + +from beanstalk_worker import task_decorator +from hat.audit import models as audit_models +from iaso.models import Task, Instance +from iaso.utils.gis import convert_2d_point_to_3d +from iaso.utils.models.common import check_instance_bulk_gps_push + +logger = getLogger(__name__) + + +def push_single_instance_gps_to_org_unit(user: Optional[User], instance: Instance): + org_unit = instance.org_unit + original_copy = deepcopy(org_unit) + org_unit.location = convert_2d_point_to_3d(instance.location) if instance.location else None + org_unit.save() + if not original_copy.location: + logger.info(f"updating {org_unit.name} {org_unit.id} with {org_unit.location}") + else: + logger.info( + f"updating {org_unit.name} {org_unit.id} - overwriting {original_copy.location} with {org_unit.location}" + ) + audit_models.log_modification(original_copy, org_unit, source=audit_models.INSTANCE_API_BULK, user=user) + + +@task_decorator(task_name="instance_bulk_gps_push") +def instance_bulk_gps_push( + select_all: bool, + selected_ids: List[int], + unselected_ids: List[int], + task: Task, +): + """Background Task to bulk push instance gps to org units. + + /!\ Danger: calling this task without having received a successful response from the check_bulk_gps_push + endpoint will have unexpected results that might cause data loss. + """ + start = time() + task.report_progress_and_stop_if_killed(progress_message="Searching for Instances for pushing gps data") + + user = task.launcher + + queryset = Instance.non_deleted_objects.get_queryset().filter_for_user(user) + queryset = queryset.select_related("org_unit") + + if not select_all: + queryset = queryset.filter(pk__in=selected_ids) + else: + queryset = queryset.exclude(pk__in=unselected_ids) + + if not queryset: + raise Exception("No matching instances found") + + # Checking if any gps push can be performed with what was requested + success, errors, _ = check_instance_bulk_gps_push(queryset) + if not success: + raise Exception("Cannot proceed with the gps push due to errors: %s" % errors) + + total = queryset.count() + + with transaction.atomic(): + for index, instance in enumerate(queryset.iterator()): + res_string = "%.2f sec, processed %i instances" % (time() - start, index) + task.report_progress_and_stop_if_killed(progress_message=res_string, end_value=total, progress_value=index) + push_single_instance_gps_to_org_unit( + user, + instance, + ) + + task.report_success(message="%d modified" % total) diff --git a/iaso/tests/api/test_instances.py b/iaso/tests/api/test_instances.py index e7defdfddc..4c9c7d4a06 100644 --- a/iaso/tests/api/test_instances.py +++ b/iaso/tests/api/test_instances.py @@ -39,7 +39,7 @@ def setUpTestData(cls): cls.sw_version = sw_version cls.yoda = cls.create_user_with_profile( - username="yoda", last_name="Da", first_name="Yo", account=star_wars, permissions=["iaso_submissions"] + username="yoda", last_name="Da", first_name="Yo", account=star_wars, permissions=["iaso_submissions", "iaso_org_units"] ) cls.guest = cls.create_user_with_profile(username="guest", account=star_wars, permissions=["iaso_submissions"]) cls.supervisor = cls.create_user_with_profile( @@ -72,10 +72,10 @@ def setUpTestData(cls): version=sw_version, ) cls.jedi_council_endor = m.OrgUnit.objects.create( - name="Endor Jedi Council", source_ref="jedi_council_endor_ref" + name="Endor Jedi Council", source_ref="jedi_council_endor_ref", version=sw_version, ) cls.jedi_council_endor_region = m.OrgUnit.objects.create( - name="Endor Region Jedi Council", parent=cls.jedi_council_endor, source_ref="jedi_council_endor_region_ref" + name="Endor Region Jedi Council", parent=cls.jedi_council_endor, source_ref="jedi_council_endor_region_ref", version=sw_version, ) cls.project = m.Project.objects.create( @@ -1963,8 +1963,8 @@ def test_check_bulk_push_gps_select_all_ok(self): response_json = response.json() self.assertEqual(response_json["result"], "success") - def test_check_bulk_push_gps_select_all_error(self): - # setting gps data for instances that were not deleted + def test_check_bulk_push_gps_select_all_error_same_org_unit(self): + # changing location for some instances to have multiple hits on multiple org_units self.instance_1.org_unit = self.jedi_council_endor self.instance_2.org_unit = self.jedi_council_endor new_location = Point(1, 2, 3) @@ -1973,14 +1973,55 @@ def test_check_bulk_push_gps_select_all_error(self): instance.save() instance.refresh_from_db() + # Let's delete some instances, the result will be the same + for instance in [self.instance_6, self.instance_8]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + self.client.force_authenticate(self.yoda) response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual(response_json["error_same_org_unit"], [self.instance_1.id, self.instance_2.id, self.instance_3.id, self.instance_4.id, self.instance_5.id]) + + def test_check_bulk_push_gps_select_all_error_read_only_source(self): + # Making the source read only + self.sw_source.read_only = True + self.sw_source.save() + + # Changing some instance.org_unit so that all the results don't appear only in "error_same_org_unit" + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + for instance in [self.instance_2, self.instance_3, self.instance_8]: + instance.save() + instance.refresh_from_db() + + self.client.force_authenticate(self.yoda) + response = self.client.get(f"/api/instances/check_bulk_gps_push/") # by default, select_all = True + self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) + response_json = response.json() + self.assertEqual(response_json["result"], "errors") + # instance_6 included because it's the first one with the remaining org_unit and the queryset has a default order of "-id" + self.assertCountEqual(response_json["error_read_only_source"], [self.instance_8.id, self.instance_2.id, self.instance_3.id, self.instance_6.id]) def test_check_bulk_push_gps_select_all_warning_no_location(self): + # Changing some instance.org_unit so that all the results don't appear only in "error_same_org_unit" + self.instance_2.org_unit = self.jedi_council_endor + self.instance_3.org_unit = self.jedi_council_endor_region + self.instance_8.org_unit = self.ou_top_1 + for instance in [self.instance_2, self.instance_3, self.instance_8]: + instance.save() + instance.refresh_from_db() + + # Let's delete some instances to avoid getting "error_same_org-unit" + for instance in [self.instance_4, self.instance_5, self.instance_6, self.instance_8]: + instance.deleted_at = datetime.datetime.now() + instance.deleted = True + instance.save() + self.client.force_authenticate(self.yoda) response = self.client.get(f"/api/instances/check_bulk_gps_push/") self.assertEqual(response.status_code, status.HTTP_200_OK) @@ -2095,8 +2136,8 @@ def test_check_bulk_push_gps_selected_ids_error(self): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() # All these Instances target the same OrgUnit, so it's impossible to push gps data - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id, self.instance_3.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual(response_json["error_same_org_unit"], [self.instance_1.id, self.instance_2.id, self.instance_3.id]) def test_check_bulk_push_gps_selected_ids_error_unknown_id(self): self.client.force_authenticate(self.yoda) @@ -2254,8 +2295,8 @@ def test_check_bulk_push_gps_unselected_ids_error(self): ) self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) response_json = response.json() - self.assertEqual(response_json["result"], "error") - self.assertCountEqual(response_json["error_ids"], [self.instance_1.id, self.instance_2.id]) + self.assertEqual(response_json["result"], "errors") + self.assertCountEqual(response_json["error_same_org_unit"], [self.instance_1.id, self.instance_2.id]) def test_check_bulk_push_gps_unselected_ids_error_unknown_id(self): self.client.force_authenticate(self.yoda) diff --git a/iaso/tests/tasks/test_instance_bulk_gps_push.py b/iaso/tests/tasks/test_instance_bulk_gps_push.py new file mode 100644 index 0000000000..ae323519e8 --- /dev/null +++ b/iaso/tests/tasks/test_instance_bulk_gps_push.py @@ -0,0 +1,392 @@ +from django.contrib.auth.models import Permission +from django.contrib.contenttypes.models import ContentType +from django.contrib.gis.geos import Point +from rest_framework import status + +from hat.menupermissions import models as am +from iaso import models as m +from iaso.models import Task, QUEUED +from iaso.tests.tasks.task_api_test_case import TaskAPITestCase + + +class InstanceBulkPushGpsAPITestCase(TaskAPITestCase): + BASE_URL = "/api/tasks/create/instancebulkgpspush/" + + @classmethod + def setUpTestData(cls): + # Preparing account, data source, project, users... + cls.account, cls.data_source, cls.source_version, cls.project = cls.create_account_datasource_version_project( + "source", "account", "project" + ) + cls.user, cls.anon_user, cls.user_no_perms = cls.create_base_users( + cls.account, ["iaso_submissions", "iaso_org_units"] + ) + + # Preparing org units & locations + cls.org_unit_type = m.OrgUnitType.objects.create(name="Org unit type", short_name="OUT") + cls.org_unit_type.projects.add(cls.project) + cls.org_unit_no_location = m.OrgUnit.objects.create( + name="No location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + cls.default_location = Point(x=4, y=50, z=100, srid=4326) + cls.other_location = Point(x=2, y=-50, z=100, srid=4326) + cls.org_unit_with_default_location = m.OrgUnit.objects.create( + name="Default location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + location=cls.default_location, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + cls.org_unit_with_other_location = m.OrgUnit.objects.create( + name="Other location", + source_ref="org unit", + validation_status=m.OrgUnit.VALIDATION_VALID, + location=cls.other_location, + version=cls.source_version, + org_unit_type=cls.org_unit_type, + ) + + # Preparing instances - all linked to org_unit_without_location + cls.form = m.Form.objects.create(name="form", period_type=m.MONTH, single_per_period=True) + cls.instance_without_location = cls.create_form_instance( + form=cls.form, + period="202001", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="noLoc", + ) + cls.instance_with_default_location = cls.create_form_instance( + form=cls.form, + period="202002", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="defaultLoc", + location=cls.default_location, + ) + cls.instance_with_other_location = cls.create_form_instance( + form=cls.form, + period="202003", + org_unit=cls.org_unit_no_location, + project=cls.project, + created_by=cls.user, + export_id="otherLoc", + location=cls.other_location, + ) + + def test_ok(self): + """POST /api/tasks/create/instancebulkgpspush/ without any error nor warning""" + + # Setting up one more instance and orgunit + new_org_unit = m.OrgUnit.objects.create( + name="new org unit", + org_unit_type=self.org_unit_type, + validation_status=m.OrgUnit.VALIDATION_VALID, + version=self.source_version, + source_ref="new org unit", + ) + new_instance = m.Instance.objects.create( + org_unit=new_org_unit, + form=self.form, + period="202004", + project=self.project, + created_by=self.user, + export_id="instance4", + ) + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_without_location.id, new_instance.id], + }, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_201_CREATED) + + def test_not_logged_in(self): + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_401_UNAUTHORIZED) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_no_permission_instances(self): + """POST /api/tasks/create/instancebulkgpspush/ without instances permissions""" + # Adding org unit permission to user + content_type = ContentType.objects.get_for_model(am.CustomPermissionSupport) + self.user_no_perms.user_permissions.add( + Permission.objects.filter(codename="iaso_org_units", content_type=content_type).first().id + ) + + self.client.force_authenticate(self.user_no_perms) + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_403_FORBIDDEN) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_no_permission_org_units(self): + """POST /api/tasks/create/instancebulkgpspush/ without orgunit permissions""" + # Adding instances permission to user + content_type = ContentType.objects.get_for_model(am.CustomPermissionSupport) + self.user_no_perms.user_permissions.add( + Permission.objects.filter(codename="iaso_submissions", content_type=content_type).first().id + ) + + self.client.force_authenticate(self.user_no_perms) + response = self.client.post( + self.BASE_URL, + format="json", + ) + self.assertJSONResponse(response, status.HTTP_403_FORBIDDEN) + self.assertEqual(Task.objects.filter(status=QUEUED).count(), 0) + + def test_instance_ids_wrong_account(self): + """POST /api/tasks/create/instancebulkgpspush/ with instance IDs from another account""" + # Preparing new setup + new_account, new_data_source, _, new_project = self.create_account_datasource_version_project( + "new source", "new account", "new project" + ) + new_user = self.create_user_with_profile( + username="new user", account=new_account, permissions=["iaso_submissions", "iaso_org_units"] + ) + new_org_unit = m.OrgUnit.objects.create( + name="New Org Unit", source_ref="new org unit", validation_status="VALID" + ) + new_form = m.Form.objects.create(name="new form", period_type=m.MONTH, single_per_period=True) + _ = self.create_form_instance( + form=new_form, + period="202001", + org_unit=new_org_unit, + project=new_project, + created_by=new_user, + export_id="Vzhn0nceudr", + location=Point(1, 2, 3, 4326), + ) + + self.client.force_authenticate(new_user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_without_location.id, self.instance_with_default_location.id], + }, + format="json", + ) + + # Task is successfully created but will fail once it starts + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, new_user) + + # Let's run the task to see the error + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + self.assertEqual( + task.result["message"], "No matching instances found" + ) # Because the instance IDs are from another account + + # Making sure that nothing changed in both accounts + self.assertIsNone(new_org_unit.location) + self.assertIsNone(self.org_unit_no_location.location) + self.assertEqual(self.org_unit_with_default_location.location, self.default_location) + + def test_overwrite_existing_location(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that overwrite existing org unit locations""" + # Setting a new location for both org_units + location = Point(42, 69, 420, 4326) + for org_unit in [self.org_unit_with_default_location, self.org_unit_with_other_location]: + org_unit.location = location + org_unit.save() + + # Linking both instances to these org_units + self.instance_with_default_location.org_unit = self.org_unit_with_default_location + self.instance_with_default_location.save() + self.instance_with_other_location.org_unit = self.org_unit_with_other_location + self.instance_with_other_location.save() + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + data={ + "select_all": False, + "selected_ids": [self.instance_with_default_location.id, self.instance_with_other_location.id], + }, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be a success + self.runAndValidateTask(task, "SUCCESS") + + self.org_unit_with_default_location.refresh_from_db() + self.org_unit_with_other_location.refresh_from_db() + self.assertEqualLocations(self.org_unit_with_default_location.location, self.default_location) + self.assertEqualLocations(self.org_unit_with_other_location.location, self.other_location) + self.assertIsNone(self.org_unit_no_location.location) + + def test_no_location(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that don't have any location defined""" + # Let's create another instance without a location, but this time it's linked to self.org_unit_with_default_location + _ = m.Instance.objects.create( + form=self.form, + period="202001", + org_unit=self.org_unit_with_default_location, + project=self.project, + created_by=self.user, + export_id="noLoc", + ) + + self.client.force_authenticate(self.user) + # For a change, let's select everything, but remove the two instances with a location + # (= it's the same as electing both instances without location) + response = self.client.post( + self.BASE_URL, + data={ + "unselected_ids": [self.instance_with_default_location.id, self.instance_with_other_location.id], + }, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be a success + self.runAndValidateTask(task, "SUCCESS") + + self.org_unit_with_default_location.refresh_from_db() + self.org_unit_no_location.refresh_from_db() + self.assertIsNone(self.org_unit_with_default_location.location) # Got overwritten by None + self.assertIsNone(self.org_unit_no_location.location) # Still None + self.assertEqualLocations(self.org_unit_with_other_location.location, self.other_location) # Not updated + + def test_multiple_updates_same_org_unit(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that target the same orgunit""" + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_same_org_unit", result) + for instance in [self.instance_without_location, self.instance_with_other_location, self.instance_with_default_location]: + self.assertIn(str(instance.id), result) + + def test_read_only_data_source(self): + """POST /api/tasks/create/instancebulkgpspush/ with instances that target orgunits which are part of a read-only data source""" + self.data_source.read_only = True + self.data_source.save() + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_read_only_source", result) + for instance in [self.instance_without_location, self.instance_with_other_location, self.instance_with_default_location]: + self.assertIn(str(instance.id), result) + + def test_all_errors(self): + """POST /api/tasks/create/instancebulkgpspush/ all errors are triggered""" + # Preparing a new read-only data source + new_data_source = m.DataSource.objects.create(name="new data source", read_only=True) + new_version = m.SourceVersion.objects.create(data_source=new_data_source, number=2) + new_data_source.projects.set([self.project]) + new_org_unit = m.OrgUnit.objects.create( + name="new org unit", + org_unit_type=self.org_unit_type, + validation_status=m.OrgUnit.VALIDATION_VALID, + version=new_version, + source_ref="new org unit", + ) + new_instance = m.Instance.objects.create( + org_unit=new_org_unit, + form=self.form, + period="202004", + project=self.project, + created_by=self.user, + export_id="instance4", + location=self.default_location, + ) + + # Changing this org unit so that it does not trigger error_same_org_unit + self.instance_with_default_location.org_unit = self.org_unit_with_default_location + self.instance_with_default_location.save() + + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + response_json = self.assertJSONResponse(response, status.HTTP_201_CREATED) + task = self.assertValidTaskAndInDB(response_json["task"], status="QUEUED", name="instance_bulk_gps_push") + self.assertEqual(task.launcher, self.user) + + # It should be an error because the check function returned errors + self.runAndValidateTask(task, "ERRORED") + task.refresh_from_db() + result = task.result["message"] + self.assertIn("Cannot proceed with the gps push due to errors", result) + self.assertIn("error_read_only_source", result) + self.assertIn("error_same_org_unit", result) + for instance in [self.instance_without_location, self.instance_with_other_location, new_instance]: + self.assertIn(str(instance.id), result) # Instead, we should probably check in which error they end up + self.assertNotIn(str(self.instance_with_default_location.id), result) + + def test_task_kill(self): + """Launch the task and then kill it + Note this actually doesn't work if it's killed while in the transaction part. + """ + self.client.force_authenticate(self.user) + response = self.client.post( + self.BASE_URL, + format="json", + ) + + data = self.assertJSONResponse(response, status.HTTP_201_CREATED) + self.assertValidTaskAndInDB(data["task"]) + + task = Task.objects.get(id=data["task"]["id"]) + task.should_be_killed = True + task.save() + + self.runAndValidateTask(task, "KILLED") + + def assertEqualLocations(self, point_1: Point, point_2: Point): + self.assertEqual(point_1.x, point_2.x) + self.assertEqual(point_1.y, point_2.y) + self.assertEqual(point_1.z, point_2.z) + self.assertEqual(point_1.srid, point_2.srid) diff --git a/iaso/urls.py b/iaso/urls.py index e0d023b409..a831b2fbc7 100644 --- a/iaso/urls.py +++ b/iaso/urls.py @@ -93,6 +93,7 @@ from .api.tasks import TaskSourceViewSet from .api.tasks.create.export_mobile_setup import ExportMobileSetupViewSet from .api.tasks.create.import_gpkg import ImportGPKGViewSet +from .api.tasks.create.instance_bulk_gps_push import InstanceBulkGpsPushViewSet from .api.tasks.create.org_unit_bulk_location_set import OrgUnitsBulkLocationSet from .api.user_roles import UserRolesViewSet from .api.workflows.changes import WorkflowChangeViewSet @@ -168,6 +169,7 @@ router.register(r"tasks/create/orgunitsbulklocationset", OrgUnitsBulkLocationSet, basename="orgunitsbulklocationset") router.register(r"tasks/create/importgpkg", ImportGPKGViewSet, basename="importgpkg") router.register(r"tasks/create/exportmobilesetup", ExportMobileSetupViewSet, basename="exportmobilesetup") +router.register(r"tasks/create/instancebulkgpspush", InstanceBulkGpsPushViewSet, basename="instancebulkgpspush") router.register(r"tasks", TaskSourceViewSet, basename="tasks") router.register(r"comments", CommentViewSet, basename="comments") router.register(r"entities", EntityViewSet, basename="entity") diff --git a/iaso/utils/models/common.py b/iaso/utils/models/common.py index 77dd3c031c..d6485d0e0d 100644 --- a/iaso/utils/models/common.py +++ b/iaso/utils/models/common.py @@ -1,3 +1,7 @@ +from typing import Dict, List + +from django.db.models import QuerySet + from iaso.models.base import User @@ -23,3 +27,74 @@ def get_org_unit_parents_ref(field_name, org_unit, parent_source_ref_field_names if parent_ref: return f"iaso#{parent_ref}" return None + +def check_instance_bulk_gps_push(queryset: QuerySet) -> (bool, Dict[str, List[int]], Dict[str, List[int]]): + """ + Determines if there are any warnings or errors if the given Instances were to push their own location to their OrgUnit. + + There are 2 types of warnings: + - warning_no_location: if an Instance doesn't have any location + - warning_overwrite: if the Instance's OrgUnit already has a location + The gps push can be performed even if there are any warnings, keeping in mind the consequences. + + There are 2 types of errors: + - error_same_org_unit: if there are multiple Instances in the given queryset that share the same OrgUnit + - error_read_only_source: if any Instance's OrgUnit is part of a read-only DataSource + The gps push cannot be performed if there are any errors. + """ + # Variables used for warnings + set_org_units_ids = set() + overwrite_ids = [] + no_location_ids = [] + + # Variables used for errors + org_units_to_instances_dict = {} + read_only_data_sources = [] + + for instance in queryset: + # First, let's check for potential errors + org_unit = instance.org_unit + if org_unit.id in org_units_to_instances_dict: + # we can't push this instance's location since there was another instance linked to this OrgUnit + org_units_to_instances_dict[org_unit.id].append(instance.id) + continue + else: + org_units_to_instances_dict[org_unit.id] = [instance.id] + + if org_unit.version and org_unit.version.data_source.read_only: + read_only_data_sources.append(instance.id) + continue + + # Then, let's check for potential warnings + if not instance.location: + no_location_ids.append(instance.id) # there is nothing to push to the OrgUnit + continue + + set_org_units_ids.add(org_unit.id) + if org_unit.location or org_unit.geom: + overwrite_ids.append(instance.id) # if the user proceeds, he will erase existing location + continue + + # Before returning, we need to check if we've had multiple hits on an OrgUnit + error_same_org_unit_ids = _check_bulk_gps_repeated_org_units(org_units_to_instances_dict) + + success: bool = not read_only_data_sources and not error_same_org_unit_ids + errors = {} + if read_only_data_sources: + errors["error_read_only_source"] = read_only_data_sources + if error_same_org_unit_ids: + errors["error_same_org_unit"] = error_same_org_unit_ids + warnings = {} + if no_location_ids: + warnings["warning_no_location"] = no_location_ids + if overwrite_ids: + warnings["warning_overwrite"] = overwrite_ids + + return success, errors, warnings + +def _check_bulk_gps_repeated_org_units(org_units_to_instance_ids: Dict[int, List[int]]) -> List[int]: + error_instance_ids = [] + for _, instance_ids in org_units_to_instance_ids.items(): + if len(instance_ids) >= 2: + error_instance_ids.extend(instance_ids) + return error_instance_ids \ No newline at end of file