From d0807251b97d5c6de69d02bd90bbee2d41a1dca6 Mon Sep 17 00:00:00 2001 From: Alexander Griffen Date: Thu, 28 Mar 2024 17:10:28 +0000 Subject: [PATCH 1/4] WIP: Update air quality data to use more recent data and also include WHO standards This currently does not show up in the UI. Additionally, the following tasks need to be completed on this work: - Finding a way to show the WHO standards in the UI - Possibly finding more WHO standards, as I only could find them for some of the pollutants --- .../commands/import_air_quality_data.py | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/hub/management/commands/import_air_quality_data.py b/hub/management/commands/import_air_quality_data.py index 65ef0409b..efb0ec866 100644 --- a/hub/management/commands/import_air_quality_data.py +++ b/hub/management/commands/import_air_quality_data.py @@ -25,7 +25,7 @@ class Command(BaseCommand): "data_type": "float", "category": "place", "source_label": "Data from Defra.", - "release_date": "2021", + "release_date": "2022", "is_range": True, "source": source_url, "source_type": "csv", @@ -42,47 +42,50 @@ class Command(BaseCommand): "pm_10": { "pollutant": "PM10", "metric": "Annual mean", - "header_label": "pm102021g", + "header_label": "pm102022g", + "WHO_AQG": "15", "comments": "Gravimetric units", - "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mappm102021g.csv", + "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mappm102022g.csv", }, "pm_2_5": { "pollutant": "PM2.5", "metric": "Annual mean", - "header_label": "pm252021g", - "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mappm252021g.csv", + "header_label": "pm252022g", + "WHO_AQG": "5", + "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mappm252022g.csv", }, "no_2": { "pollutant": "NO2", "metric": "Annual mean", - "header_label": "no22021", - "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapno22021.csv", + "header_label": "no22022", + "WHO_AQG": "10", + "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapno22022.csv", }, "no_x": { "pollutant": "NOx", "metric": "Annual mean", - "header_label": "nox2021", + "header_label": "nox2022", "comments": "µg m\u207B\u00B3 (NO\u2093 as NO\u2082)", - "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapnox2021.csv", + "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapnox2022.csv", }, "so_2": { "pollutant": "SO2", "metric": "Annual mean", - "header_label": "so22021", - "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapso22021.csv", + "header_label": "so22022", + "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapso22022.csv", }, "ozone": { "pollutant": "Ozone", "metric": "DGT120", - "header_label": "dgt12021", + "header_label": "dgt12022", "comments": "number of days on which the daily max 8-hr concentration is greater than 120 µg m\u207B\u00B3", - "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapdgt12021.csv", + "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapdgt12022.csv", }, "benzene": { "pollutant": "Benzene", "metric": "Annual mean", - "header_label": "bz2021", - "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapbz2021.csv", + "header_label": "bz2022", + "csv_link": "https://uk-air.defra.gov.uk/datastore/pcm/mapbz2022.csv", }, } @@ -160,7 +163,7 @@ def get_dataframe(self): skiprows=5, na_values="MISSING", dtype={"gridcode": "int", metadata["header_label"]: "float"}, - ).rename(columns={metadata["header_label"]: label}) + ).rename(columns={metadata["header_label"]: label}).assign(WHO_AQG=lambda x: metadata.get('WHO_AQG', None)) ) print("Transforming data") From 008e85cc7dd8d12304d46bf1b525a3294d56a7fb Mon Sep 17 00:00:00 2001 From: Struan Donald Date: Mon, 1 Apr 2024 16:28:10 +0100 Subject: [PATCH 2/4] fixup! WIP: Update air quality data to use more recent data and also include WHO standards --- hub/management/commands/import_air_quality_data.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/hub/management/commands/import_air_quality_data.py b/hub/management/commands/import_air_quality_data.py index efb0ec866..034958bda 100644 --- a/hub/management/commands/import_air_quality_data.py +++ b/hub/management/commands/import_air_quality_data.py @@ -8,8 +8,10 @@ from hub.models import Area, AreaData, DataSet, DataType +from .base_importers import BaseAreaImportCommand -class Command(BaseCommand): + +class Command(BaseAreaImportCommand): help = "Import air-pollution data" source_url = "https://uk-air.defra.gov.uk/data/modelling-data" @@ -107,6 +109,8 @@ def create_data_types(self, df): data_set, created = DataSet.objects.update_or_create( name="constituency_air_quality", defaults=self.defaults ) + data_set.areas_available.add(self.get_area_type()) + data_types = [] for col in tqdm(df.columns, disable=self._quiet): label = self.in_files[col]["pollutant"] @@ -117,6 +121,7 @@ def create_data_types(self, df): data_type, created = DataType.objects.update_or_create( data_set=data_set, name=f"air_quality_{col}", + area_type=self.get_area_type(), defaults={ "data_type": "float", "label": label, From 27fcd838c262b04c8759a06833cf2e3227e08148 Mon Sep 17 00:00:00 2001 From: Struan Donald Date: Mon, 1 Apr 2024 16:28:54 +0100 Subject: [PATCH 3/4] fixup! WIP: Update air quality data to use more recent data and also include WHO standards --- hub/management/commands/import_air_quality_data.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hub/management/commands/import_air_quality_data.py b/hub/management/commands/import_air_quality_data.py index 034958bda..992a1a11f 100644 --- a/hub/management/commands/import_air_quality_data.py +++ b/hub/management/commands/import_air_quality_data.py @@ -168,7 +168,9 @@ def get_dataframe(self): skiprows=5, na_values="MISSING", dtype={"gridcode": "int", metadata["header_label"]: "float"}, - ).rename(columns={metadata["header_label"]: label}).assign(WHO_AQG=lambda x: metadata.get('WHO_AQG', None)) + ) + .rename(columns={metadata["header_label"]: label}) + .assign(WHO_AQG=lambda x: metadata.get("WHO_AQG", None)) ) print("Transforming data") From 888c3bf427c311a0b7ebe64eb240139418a57833 Mon Sep 17 00:00:00 2001 From: Struan Donald Date: Mon, 1 Apr 2024 16:29:40 +0100 Subject: [PATCH 4/4] fixup! WIP: Update air quality data to use more recent data and also include WHO standards --- hub/management/commands/import_air_quality_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hub/management/commands/import_air_quality_data.py b/hub/management/commands/import_air_quality_data.py index 992a1a11f..880e10107 100644 --- a/hub/management/commands/import_air_quality_data.py +++ b/hub/management/commands/import_air_quality_data.py @@ -1,7 +1,6 @@ from functools import reduce from django.conf import settings -from django.core.management.base import BaseCommand import pandas as pd from tqdm import tqdm