fix CI and force all files to xlsx instead of xls

iaso/odk/validator.py

@@ -55,21 +55,31 @@ def is_repeat_group(question):
     return question.get("type") == "begin repeat" or question.get("type") == "begin_repeat"
 
 
-def get_list_name(question):
+def get_list_name_from_select(question):
     q_type = question.get("type")
+    if "select one from " in q_type:
+        return q_type.split("select one from ")[1]
     if "select_one" in q_type:
         return q_type.split("select_one ")[1]
     if "select one" in q_type:
+        import pdb
+
+        pdb.set_trace()
         return q_type.split("select one ")[1]
     return None
 
 
-def index_by_field(collection, field_name):
+def get_list_name_from_choice(choice):
+    return choice.get("list_name") or choice.get("list name")
+
+
+def group_by_lambda(collection, field_name_lambda):
     collection_by_name = {}
 
     for q in collection:
-        if q.get(field_name):
-            collection_by_name.setdefault(q[field_name], []).append(q)
+        field_value = field_name_lambda(q)
+        if field_value:
+            collection_by_name.setdefault(field_value, []).append(q)
 
     return collection_by_name
 
@@ -82,13 +92,13 @@ def validate_xls_form(xls_file):
     choices_rows = parse_sheet(excel_file, "choices")
 
     questions = [q for q in question_rows if q.get("type") and not is_end_group(q) and not is_end_repeat(q)]
-    choices = [c for c in choices_rows if c.get("list_name")]
+    choices = [c for c in choices_rows if get_list_name_from_choice(c)]
 
     validation_errors = []
 
-    questions_by_name = index_by_field(questions, "name")
+    questions_by_name = group_by_lambda(questions, lambda x: x.get("name"))
 
-    choices_by_list_name = index_by_field(choices, "list_name")
+    choices_by_list_name = group_by_lambda(choices, lambda x: get_list_name_from_choice(x))
 
     has_some_repeat_groups = len([q for q in question_rows if is_repeat_group(q)]) > 0
 
@@ -119,7 +129,7 @@ def validate_xls_form(xls_file):
     select_one_questions = [q for q in questions if is_select_one(q)]
 
     for select_one_q in select_one_questions:
-        list_name = get_list_name(select_one_q)
+        list_name = get_list_name_from_select(select_one_q)
         if list_name not in choices_by_list_name:
             validation_errors.append(
                 {

iaso/tests/api/test_form_versions.py

@@ -52,7 +52,7 @@ def setUpTestData(cls):
         cls.form_1.save()
         form_2_file_mock = mock.MagicMock(spec=File)
         form_2_file_mock.name = "test.xml"
-        with open("iaso/tests/fixtures/odk_form_valid_no_settings.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_no_settings.xlsx", "rb") as xls_file:
             cls.form_2.form_versions.create(
                 file=form_2_file_mock, xls_file=UploadedFile(xls_file), version_id="2020022401"
             )
@@ -197,7 +197,7 @@ def test_form_versions_create_ok_first_version(self):
         """POST /form-versions/ happy path (first version)"""
 
         self.client.force_authenticate(self.yoda)
-        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file:
             response = self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_1.id, "xls_file": xls_file},
@@ -217,7 +217,7 @@ def test_form_versions_create_ok_first_version(self):
         self.assertRegex(created_version.file.name, r"forms/new_land_speeder_concept_2020022401(.*).xml")
         self.assertIsInstance(created_version.xls_file, File)
         self.assertGreater(created_version.xls_file.size, 100)
-        self.assertEqual(created_version.xls_file.name, "forms/new_land_speeder_concept_2020022401.xls")
+        self.assertEqual(created_version.xls_file.name, "forms/new_land_speeder_concept_2020022401.xlsx")
 
         version_form = created_version.form
         self.assertEqual("sample1", version_form.form_id)
@@ -226,7 +226,7 @@ def test_form_versions_create_ok_second_version(self):
         """POST /form-versions/ happy path (second version)"""
 
         self.client.force_authenticate(self.yoda)
-        with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xlsx", "rb") as xls_file:
             response = self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_2.id, "xls_file": xls_file},
@@ -271,7 +271,7 @@ def test_form_versions_create_ok_second_version_with_mappings(self):
             },
         )
 
-        with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022402.xlsx", "rb") as xls_file:
             response = self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_2.id, "xls_file": xls_file},
@@ -300,7 +300,7 @@ def test_form_versions_create_invalid_xls_form_id_1(self):
         """POST /form-versions/ with a form_id that already exists within the account (for a different form)"""
 
         self.client.force_authenticate(self.yoda)
-        with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022401.xlsx", "rb") as xls_file:
             response = self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_1.id, "xls_file": xls_file},
@@ -314,7 +314,7 @@ def test_form_versions_create_invalid_xls_form_id_2(self):
         """POST /form-versions/ attempt to create a second version with a different form_id"""
 
         self.client.force_authenticate(self.yoda)
-        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022402.xlsx", "rb") as xls_file:
             response = self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_2.id, "xls_file": xls_file},
@@ -328,7 +328,7 @@ def test_form_versions_create_invalid_xls_version(self):
         """POST /form-versions/ attempt to create a second version with a version inferior to the previous one"""
 
         self.client.force_authenticate(self.yoda)
-        with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample2_2020022301.xlsx", "rb") as xls_file:
             response = self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_2.id, "xls_file": xls_file},
@@ -344,7 +344,7 @@ def test_form_versions_create_invalid_xls_file(self):
         """POST /form-versions/ with invalid XLS file"""
 
         self.client.force_authenticate(self.yoda)
-        with open("iaso/tests/fixtures/odk_form_blatantly_invalid.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_blatantly_invalid.xlsx", "rb") as xls_file:
             response = self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_1.id, "xls_file": xls_file},
@@ -372,7 +372,7 @@ def test_form_versions_create_no_xls_file(self):
     def test_form_versions_create_no_auth(self):
         """POST /form-versions/ , without auth -> we expect a 401 error"""
 
-        with open("iaso/tests/fixtures/odk_form_valid_no_settings.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_no_settings.xlsx", "rb") as xls_file:
             response = self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_1.id, "xls_file": xls_file},

iaso/tests/api/test_mappingversions.py

@@ -327,7 +327,7 @@ def test_mappingversions_list_filters_project_ids(self):
         self.assertEqual(len(resp.json()["mapping_versions"]), 1)
 
     def create_form_version(self):
-        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file:
             self.client.post(
                 f"/api/formversions/",
                 data={"form_id": self.form_1.id, "xls_file": xls_file},

iaso/tests/fixtures/.~lock.odk_form_blatantly_invalid.xlsx#

@@ -0,0 +1 @@
+,smestach,bebop,18.12.2024 15:07,file:///home/smestach/.config/libreoffice/4;

iaso/tests/fixtures/.~lock.odk_form_valid_sample1_2020022401.xlsx#

@@ -0,0 +1 @@
+,smestach,bebop,18.12.2024 14:55,file:///home/smestach/.config/libreoffice/4;

iaso/tests/fixtures/.~lock.odk_form_valid_sample2_2020022402.xlsx#

@@ -0,0 +1 @@
+,smestach,bebop,18.12.2024 14:54,file:///home/smestach/.config/libreoffice/4;

iaso/tests/odk/test_parsing.py

@@ -9,7 +9,7 @@
 
 class ParsingTestCase(SimpleTestCase):
     def test_parse_xls_form_valid(self):
-        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file:
             survey = parse_xls_form(xls_file)
 
         self.assertIsInstance(survey, Survey)
@@ -23,7 +23,7 @@ def test_parse_xls_form_valid(self):
         self.assertEqual(survey.version, "2020022401")
 
     def test_parse_xls_form_valid_no_settings(self):
-        with open("iaso/tests/fixtures/odk_form_valid_no_settings.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_no_settings.xlsx", "rb") as xls_file:
             survey = parse_xls_form(xls_file)
 
         self.assertIsInstance(survey, Survey)
@@ -40,15 +40,15 @@ def test_parse_xls_form_invalid_version(self):
             self.assertEqual(str(cm.exception), "Invalid XLS file: Invalid version (must be a string of 1-10 numbers).")
 
     def test_parse_xls_form_invalid_version_inferior_to_previous(self):
-        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file:
             with self.assertRaises(ParsingError) as cm:
                 parse_xls_form(xls_file, previous_version="2020022501")
             self.assertEqual(
                 str(cm.exception), "Invalid XLS file: Parsed version should be greater than previous version."
             )
 
     def test_parse_xls_form_invalid_version_same_as_previous(self):
-        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xls", "rb") as xls_file:
+        with open("iaso/tests/fixtures/odk_form_valid_sample1_2020022401.xlsx", "rb") as xls_file:
             with self.assertRaises(ParsingError) as cm:
                 parse_xls_form(xls_file, previous_version="2020022401")
             self.assertEqual(