diff --git a/iaso/odk/validator.py b/iaso/odk/validator.py index 45c235e730..9f85d2be0f 100644 --- a/iaso/odk/validator.py +++ b/iaso/odk/validator.py @@ -55,21 +55,31 @@ def is_repeat_group(question): return question.get("type") == "begin repeat" or question.get("type") == "begin_repeat" -def get_list_name(question): +def get_list_name_from_select(question): q_type = question.get("type") + if "select one from " in q_type: + return q_type.split("select one from ")[1] if "select_one" in q_type: return q_type.split("select_one ")[1] if "select one" in q_type: + import pdb + + pdb.set_trace() return q_type.split("select one ")[1] return None -def index_by_field(collection, field_name): +def get_list_name_from_choice(choice): + return choice.get("list_name") or choice.get("list name") + + +def group_by_lambda(collection, field_name_lambda): collection_by_name = {} for q in collection: - if q.get(field_name): - collection_by_name.setdefault(q[field_name], []).append(q) + field_value = field_name_lambda(q) + if field_value: + collection_by_name.setdefault(field_value, []).append(q) return collection_by_name @@ -82,13 +92,13 @@ def validate_xls_form(xls_file): choices_rows = parse_sheet(excel_file, "choices") questions = [q for q in question_rows if q.get("type") and not is_end_group(q) and not is_end_repeat(q)] - choices = [c for c in choices_rows if c.get("list_name")] + choices = [c for c in choices_rows if get_list_name_from_choice(c)] validation_errors = [] - questions_by_name = index_by_field(questions, "name") + questions_by_name = group_by_lambda(questions, lambda x: x.get("name")) - choices_by_list_name = index_by_field(choices, "list_name") + choices_by_list_name = group_by_lambda(choices, lambda x: get_list_name_from_choice(x)) has_some_repeat_groups = len([q for q in question_rows if is_repeat_group(q)]) > 0 @@ -119,7 +129,7 @@ def validate_xls_form(xls_file): select_one_questions = [q for q in questions if is_select_one(q)] for select_one_q in select_one_questions: - list_name = get_list_name(select_one_q) + list_name = get_list_name_from_select(select_one_q) if list_name not in choices_by_list_name: validation_errors.append( { diff --git a/iaso/tests/api/test_form_versions.py b/iaso/tests/api/test_form_versions.py index 65d2a4a3aa..6edb5e250d 100644 --- a/iaso/tests/api/test_form_versions.py +++ b/iaso/tests/api/test_form_versions.py @@ -52,7 +52,7 @@ def setUpTestData(cls): cls.form_1.save() form_2_file_mock = mock.MagicMock(spec=File) form_2_file_mock.name = "test.xml" - with open("iaso/tests/fixtures/odk_form_valid_no_settings.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_no_settings.xlsx", "rb") as xls_file: cls.form_2.form_versions.create( file=form_2_file_mock, xls_file=UploadedFile(xls_file), version_id="2020022401" ) @@ -197,7 +197,7 @@ def test_form_versions_create_ok_first_version(self): """POST /form-versions/ happy path (first version)""" self.client.force_authenticate(self.yoda) - with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file: response = self.client.post( f"/api/formversions/", data={"form_id": self.form_1.id, "xls_file": xls_file}, @@ -217,7 +217,7 @@ def test_form_versions_create_ok_first_version(self): self.assertRegex(created_version.file.name, r"forms/new_land_speeder_concept_2020022401(.*).xml") self.assertIsInstance(created_version.xls_file, File) self.assertGreater(created_version.xls_file.size, 100) - self.assertEqual(created_version.xls_file.name, "forms/new_land_speeder_concept_2020022401.xls") + self.assertEqual(created_version.xls_file.name, "forms/new_land_speeder_concept_2020022401.xlsx") version_form = created_version.form self.assertEqual("sample1", version_form.form_id) @@ -226,7 +226,7 @@ def test_form_versions_create_ok_second_version(self): """POST /form-versions/ happy path (second version)""" self.client.force_authenticate(self.yoda) - with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xlsx", "rb") as xls_file: response = self.client.post( f"/api/formversions/", data={"form_id": self.form_2.id, "xls_file": xls_file}, @@ -271,7 +271,7 @@ def test_form_versions_create_ok_second_version_with_mappings(self): }, ) - with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xlsx", "rb") as xls_file: response = self.client.post( f"/api/formversions/", data={"form_id": self.form_2.id, "xls_file": xls_file}, @@ -300,7 +300,7 @@ def test_form_versions_create_invalid_xls_form_id_1(self): """POST /form-versions/ with a form_id that already exists within the account (for a different form)""" self.client.force_authenticate(self.yoda) - with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xlsx", "rb") as xls_file: response = self.client.post( f"/api/formversions/", data={"form_id": self.form_1.id, "xls_file": xls_file}, @@ -314,7 +314,7 @@ def test_form_versions_create_invalid_xls_form_id_2(self): """POST /form-versions/ attempt to create a second version with a different form_id""" self.client.force_authenticate(self.yoda) - with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xlsx", "rb") as xls_file: response = self.client.post( f"/api/formversions/", data={"form_id": self.form_2.id, "xls_file": xls_file}, @@ -328,7 +328,7 @@ def test_form_versions_create_invalid_xls_version(self): """POST /form-versions/ attempt to create a second version with a version inferior to the previous one""" self.client.force_authenticate(self.yoda) - with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xlsx", "rb") as xls_file: response = self.client.post( f"/api/formversions/", data={"form_id": self.form_2.id, "xls_file": xls_file}, @@ -344,7 +344,7 @@ def test_form_versions_create_invalid_xls_file(self): """POST /form-versions/ with invalid XLS file""" self.client.force_authenticate(self.yoda) - with open("iaso/tests/fixtures/odk_form_blatantly_invalid.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_blatantly_invalid.xlsx", "rb") as xls_file: response = self.client.post( f"/api/formversions/", data={"form_id": self.form_1.id, "xls_file": xls_file}, @@ -372,7 +372,7 @@ def test_form_versions_create_no_xls_file(self): def test_form_versions_create_no_auth(self): """POST /form-versions/ , without auth -> we expect a 401 error""" - with open("iaso/tests/fixtures/odk_form_valid_no_settings.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_no_settings.xlsx", "rb") as xls_file: response = self.client.post( f"/api/formversions/", data={"form_id": self.form_1.id, "xls_file": xls_file}, diff --git a/iaso/tests/api/test_mappingversions.py b/iaso/tests/api/test_mappingversions.py index c4274eb19e..8931743207 100644 --- a/iaso/tests/api/test_mappingversions.py +++ b/iaso/tests/api/test_mappingversions.py @@ -327,7 +327,7 @@ def test_mappingversions_list_filters_project_ids(self): self.assertEqual(len(resp.json()["mapping_versions"]), 1) def create_form_version(self): - with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file: self.client.post( f"/api/formversions/", data={"form_id": self.form_1.id, "xls_file": xls_file}, diff --git a/iaso/tests/fixtures/.~lock.odk_form_blatantly_invalid.xlsx# b/iaso/tests/fixtures/.~lock.odk_form_blatantly_invalid.xlsx# new file mode 100644 index 0000000000..7f491e5a47 --- /dev/null +++ b/iaso/tests/fixtures/.~lock.odk_form_blatantly_invalid.xlsx# @@ -0,0 +1 @@ +,smestach,bebop,18.12.2024 15:07,file:///home/smestach/.config/libreoffice/4; \ No newline at end of file diff --git a/iaso/tests/fixtures/.~lock.odk_form_valid_sample1_2020022401.xlsx# b/iaso/tests/fixtures/.~lock.odk_form_valid_sample1_2020022401.xlsx# new file mode 100644 index 0000000000..eea5c989a7 --- /dev/null +++ b/iaso/tests/fixtures/.~lock.odk_form_valid_sample1_2020022401.xlsx# @@ -0,0 +1 @@ +,smestach,bebop,18.12.2024 14:55,file:///home/smestach/.config/libreoffice/4; \ No newline at end of file diff --git a/iaso/tests/fixtures/.~lock.odk_form_valid_sample2_2020022402.xlsx# b/iaso/tests/fixtures/.~lock.odk_form_valid_sample2_2020022402.xlsx# new file mode 100644 index 0000000000..1421c6db78 --- /dev/null +++ b/iaso/tests/fixtures/.~lock.odk_form_valid_sample2_2020022402.xlsx# @@ -0,0 +1 @@ +,smestach,bebop,18.12.2024 14:54,file:///home/smestach/.config/libreoffice/4; \ No newline at end of file diff --git a/iaso/tests/fixtures/odk_form_blatantly_invalid.xls b/iaso/tests/fixtures/odk_form_blatantly_invalid.xls deleted file mode 100644 index 2828c72fe3..0000000000 Binary files a/iaso/tests/fixtures/odk_form_blatantly_invalid.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_blatantly_invalid.xlsx b/iaso/tests/fixtures/odk_form_blatantly_invalid.xlsx new file mode 100644 index 0000000000..c6c1fd89c4 Binary files /dev/null and b/iaso/tests/fixtures/odk_form_blatantly_invalid.xlsx differ diff --git a/iaso/tests/fixtures/odk_form_invalid_version.xls b/iaso/tests/fixtures/odk_form_invalid_version.xls deleted file mode 100644 index 2fbd29461b..0000000000 Binary files a/iaso/tests/fixtures/odk_form_invalid_version.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_invalid_version.xlsx b/iaso/tests/fixtures/odk_form_invalid_version.xlsx new file mode 100644 index 0000000000..7d9abc1741 Binary files /dev/null and b/iaso/tests/fixtures/odk_form_invalid_version.xlsx differ diff --git a/iaso/tests/fixtures/odk_form_valid_no_settings.xls b/iaso/tests/fixtures/odk_form_valid_no_settings.xls deleted file mode 100644 index bc09dbdd1a..0000000000 Binary files a/iaso/tests/fixtures/odk_form_valid_no_settings.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_valid_no_settings.xlsx b/iaso/tests/fixtures/odk_form_valid_no_settings.xlsx new file mode 100644 index 0000000000..df53df9acf Binary files /dev/null and b/iaso/tests/fixtures/odk_form_valid_no_settings.xlsx differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls b/iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls deleted file mode 100644 index a7c38e94ab..0000000000 Binary files a/iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx b/iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx new file mode 100644 index 0000000000..0e815349d4 Binary files /dev/null and b/iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xls b/iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xls deleted file mode 100644 index 50432bf98c..0000000000 Binary files a/iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xlsx b/iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xlsx new file mode 100644 index 0000000000..af9f9a9f31 Binary files /dev/null and b/iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xlsx differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample1_no_version.xls b/iaso/tests/fixtures/odk_form_valid_sample1_no_version.xls deleted file mode 100644 index c286261a8f..0000000000 Binary files a/iaso/tests/fixtures/odk_form_valid_sample1_no_version.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample1_no_version.xlsx b/iaso/tests/fixtures/odk_form_valid_sample1_no_version.xlsx new file mode 100644 index 0000000000..3e12028f12 Binary files /dev/null and b/iaso/tests/fixtures/odk_form_valid_sample1_no_version.xlsx differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xls b/iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xls deleted file mode 100644 index b5baf5dd46..0000000000 Binary files a/iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xlsx b/iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xlsx new file mode 100644 index 0000000000..5c132980d0 Binary files /dev/null and b/iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xlsx differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xls b/iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xls deleted file mode 100644 index 3343208a3f..0000000000 Binary files a/iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xlsx b/iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xlsx new file mode 100644 index 0000000000..ead6e14e62 Binary files /dev/null and b/iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xlsx differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xls b/iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xls deleted file mode 100644 index dfb6e8c306..0000000000 Binary files a/iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xls and /dev/null differ diff --git a/iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xlsx b/iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xlsx new file mode 100644 index 0000000000..1f61b1d7fb Binary files /dev/null and b/iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xlsx differ diff --git a/iaso/tests/odk/test_parsing.py b/iaso/tests/odk/test_parsing.py index 850b0714b3..956106cebf 100644 --- a/iaso/tests/odk/test_parsing.py +++ b/iaso/tests/odk/test_parsing.py @@ -9,7 +9,7 @@ class ParsingTestCase(SimpleTestCase): def test_parse_xls_form_valid(self): - with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file: survey = parse_xls_form(xls_file) self.assertIsInstance(survey, Survey) @@ -23,7 +23,7 @@ def test_parse_xls_form_valid(self): self.assertEqual(survey.version, "2020022401") def test_parse_xls_form_valid_no_settings(self): - with open("iaso/tests/fixtures/odk_form_valid_no_settings.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_no_settings.xlsx", "rb") as xls_file: survey = parse_xls_form(xls_file) self.assertIsInstance(survey, Survey) @@ -40,7 +40,7 @@ def test_parse_xls_form_invalid_version(self): self.assertEqual(str(cm.exception), "Invalid XLS file: Invalid version (must be a string of 1-10 numbers).") def test_parse_xls_form_invalid_version_inferior_to_previous(self): - with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file: with self.assertRaises(ParsingError) as cm: parse_xls_form(xls_file, previous_version="2020022501") self.assertEqual( @@ -48,7 +48,7 @@ def test_parse_xls_form_invalid_version_inferior_to_previous(self): ) def test_parse_xls_form_invalid_version_same_as_previous(self): - with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file: + with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file: with self.assertRaises(ParsingError) as cm: parse_xls_form(xls_file, previous_version="2020022401") self.assertEqual(