diff --git a/hub/management/commands/run_all_import_scripts.py b/hub/management/commands/run_all_import_scripts.py index f63a40a81..ab2a7d530 100644 --- a/hub/management/commands/run_all_import_scripts.py +++ b/hub/management/commands/run_all_import_scripts.py @@ -8,6 +8,23 @@ class Command(BaseCommand): help = "Run all of the import scripts" + priority_imports = [ + "import_areas", + "import_mps", + "import_mps_election_results", + ] + skip_imports = [ + "import_2024_ppcs", # no longer relevant post-election + "import_air_quality_data", # ValueError because it checks for truthiness of a Pandas dataframe? + "import_christian_aid_group_locations", # JSONDecodeError because parlparse JSON file not found + "import_mps_appg_data", # hasn't been updated for Autumn 2024 APPGs + "import_mps_relevant_votes", # hasn't been updated for a while (and we import EDMs separately now) + "import_mps_standing_down_2024", # no longer relevant post-election + "import_mps_select_committee_membership", # ValueError because it checks for truthiness of a Pandas dataframe? + "import_nt_property_locations", # ValueError because it checks for truthiness of a Pandas dataframe? + "import_onward_polling_data", # JSONDecodeError because parlparse JSON file not found + ] + def get_scripts(self, *args, **options): path = "hub/management/commands" files = [f for f in listdir(path) if isfile(join(path, f))] @@ -19,10 +36,10 @@ def get_scripts(self, *args, **options): ] generators = [] importers = [] - for script in scripts: + for script in sorted(scripts): if "generate_" in script: generators.append(script) - else: + elif script not in self.skip_imports: importers.append(script) return {"generators": generators, "importers": importers} @@ -56,8 +73,7 @@ def run_importer_scripts(self, imports, *args, **options): total = str(len(imports)) i = 1 failed_imports = {} - priority_imports = ["import_areas", "import_mps", "import_mps_election_results"] - for importer in priority_imports: + for importer in self.priority_imports: imports.remove(importer) print(f"Running command: {importer} ({str(i)}/{total})") try: