From 007d38303d1ea968cf80546efb217d96a89f9f4a Mon Sep 17 00:00:00 2001 From: Alexander Griffen Date: Thu, 24 Aug 2023 16:07:56 +0100 Subject: [PATCH] DATASET: Flood risk data Also, a change to the _place_data template that means that datasets with no actual data in them do not render empty cards --- .../commands/import_flood_risk_data.py | 107 +++++++ hub/templates/hub/area/_place_data.html | 261 ++++++++++-------- 2 files changed, 248 insertions(+), 120 deletions(-) create mode 100644 hub/management/commands/import_flood_risk_data.py diff --git a/hub/management/commands/import_flood_risk_data.py b/hub/management/commands/import_flood_risk_data.py new file mode 100644 index 000000000..b2c76454d --- /dev/null +++ b/hub/management/commands/import_flood_risk_data.py @@ -0,0 +1,107 @@ +from django.conf import settings +from django.core.management.base import BaseCommand + +import pandas as pd +from tqdm import tqdm + +from hub.models import Area, AreaData, DataSet, DataType + + +class Command(BaseCommand): + help = "Import flood risk data" + + data_file = settings.BASE_DIR / "data" / "risk_of_flooding.csv" + message = "Importing constituency flood risk data" + defaults = { + "label": "Flood risk from rivers or sea", + "description": "Data relating to risk of flooding from rivers or sea, recorded in 2018", + "data_type": "percent", + "category": "place", + "source_label": "Defra", + "is_range": True, + "source_type": "csv", + "table": "areadata", + "exclude_countries": ["Northern Ireland", "Scotland", "Wales"], + "comparators": DataSet.numerical_comparators(), + "default_value": 10, + "is_shadable": False, + } + + def add_arguments(self, parser): + parser.add_argument( + "-q", "--quiet", action="store_true", help="Silence progress bars." + ) + + def handle(self, quiet=False, *args, **options): + self._quiet = quiet + df = self.get_dataframe() + self.data_types = self.create_data_types(df) + self.delete_data() + self.import_data(df) + + def create_data_types(self, df): + if not self._quiet: + self.stdout.write("Creating dataset + types") + data_set, created = DataSet.objects.update_or_create( + name="constituency_flood_risk", defaults=self.defaults + ) + data_types = [] + for col in tqdm(df.columns, disable=self._quiet): + data_type, created = DataType.objects.update_or_create( + data_set=data_set, + name=f"flood_risk_{col.lower().replace(' ', '_')}", + defaults={ + "data_type": "percent", + "label": col, + "description": f"Percentage of the constituency marked as '{col}' risk of flooding from rivers or the sea", + }, + ) + data_types.append(data_type) + + return data_types + + def import_data(self, df): + if not self._quiet: + self.stdout.write("Importing flood risk data") + for gss, row in tqdm(df.iterrows(), disable=self._quiet): + try: + area = Area.objects.get(gss=gss) + except Area.DoesNotExist: + self.stdout.write(f"Failed to find area with code {gss}") + continue + for data_type in self.data_types: + AreaData.objects.create( + data_type=data_type, + area=area, + data=row[data_type.label], + ) + for col in df.columns: + average = df[col].mean() + data_type = DataType.objects.get( + name=f"flood_risk_{col.lower().replace(' ', '_')}" + ) + data_type.average = average + data_type.save() + + def delete_data(self): + AreaData.objects.filter(data_type__in=self.data_types).delete() + + def get_dataframe(self): + df = pd.read_csv(self.data_file) + totals = ( + df.dropna()[["gss", "prob_4band"]] + .groupby("gss") + .count() + .prob_4band.to_dict() + ) + df["total"] = df.gss.apply(lambda x: totals.get(x, None)) + df = ( + df.dropna()[["gss", "prob_4band", "total"]] + .groupby("gss") + .value_counts() + .reset_index() + .rename(columns={0: "value"}) + ) + df["percentage"] = df.value / df.total * 100 + df = df.pivot(columns="prob_4band", values="percentage", index="gss").fillna(0) + return df diff --git a/hub/templates/hub/area/_place_data.html b/hub/templates/hub/area/_place_data.html index 0428ecafa..42d66601f 100644 --- a/hub/templates/hub/area/_place_data.html +++ b/hub/templates/hub/area/_place_data.html @@ -1,131 +1,152 @@ {% load humanize %} {% load hub_filters %} -
-
-
-
{{ category.name }}
- {% include "hub/area/_favourite.html" %} -
-
- {% if category.name == "Constituency age distribution" %} - - - - - - - - - - {% for range in category.data %} - - - - - - {% endfor %} - -
Age bandThis areaUK average
{{ range.label }}{{ range.value|floatformat }}{% if range.is_percentage %}%{% endif %}{{ range.average|floatformat }}{% if range.is_percentage %}%{% endif %}
- {% elif category.name == "Ethnicity" %} - - - - - - - - - - {% for range in category.data %} - - - - - - {% endfor %} - -
EthnicityThis areaUK average
{{ range.label }}{{ range.value|floatformat }}{% if range.is_percentage %}%{% endif %}{{ range.average|floatformat }}{% if range.is_percentage %}%{% endif %}
- {% elif category.name == 'Socio-Economic Status' %} - - - - - - - - - - {% for range in category.data %} - - - - - - {% endfor %} - -
StatusThis areaUK average
{{ range.label }}{{ range.value|floatformat }}%{{ range.average|floatformat }}%
- {% elif category.data.data_type.name == 'constituency_popular_petitions' %} - - - - - - - - - {% for petition in category.data.value %} - - - - - {% endfor %} - -
PetitionSignatures
{{ petition.action }}{{ petition.signatures }}
- {% elif category.name == 'Air pollution' %} - - - - - - - - - +{% if category.data %} +
+
+
+
{{ category.name }}
+ {% include "hub/area/_favourite.html" %} +
+
+ {% if category.name == "Constituency age distribution" %} +
PollutantThis areaUK average
+ + + + + + + + {% for range in category.data %} - - - - - + + + + + {% endfor %} - -
Age bandThis areaUK average
- {{ range.label|html_format_dataset_name|safe }} - {% if range.data_type.description %} - {{ range.data_type.description|escape }} - {% endif %} - {{ range.value|floatformat }}%{{ range.average|floatformat }}%
{{ range.label }}{{ range.value|floatformat }}{% if range.is_percentage %}%{% endif %}{{ range.average|floatformat }}{% if range.is_percentage %}%{% endif %}
- {% else %} - {% if category.data.is_number %} -

{{ category.data.value|floatformat }}{% if category.data.is_percentage %}%{% endif %}

- {% if category.data.average %} -

{{ category.data.average|floatformat }}{% if category.data.is_percentage %}%{% endif %} national average

- {% endif %} + + + {% elif category.name == "Ethnicity" %} + + + + + + + + + + {% for range in category.data %} + + + + + + {% endfor %} + +
EthnicityThis areaUK average
{{ range.label }}{{ range.value|floatformat }}{% if range.is_percentage %}%{% endif %}{{ range.average|floatformat }}{% if range.is_percentage %}%{% endif %}
+ {% elif category.name == 'Socio-Economic Status' %} + + + + + + + + + + {% for range in category.data %} + + + + + + {% endfor %} + +
StatusThis areaUK average
{{ range.label }}{{ range.value|floatformat }}%{{ range.average|floatformat }}%
+ {% elif category.data.data_type.name == 'constituency_popular_petitions' %} + + + + + + + + + {% for petition in category.data.value %} + + + + + {% endfor %} + +
PetitionSignatures
{{ petition.action }}{{ petition.signatures }}
+ {% elif category.name == 'Air pollution' %} + + + + + + + + + + {% for range in category.data %} + + + + + + {% endfor %} + +
PollutantThis areaUK average
+ {{ range.label|html_format_dataset_name|safe }} + {% if range.data_type.description %} + {{ range.data_type.description|escape }} + {% endif %} + {{ range.value|floatformat }}%{{ range.average|floatformat }}%
+ {% elif category.name == 'Flood risk from rivers or sea' %} + + + + + + + + + + {% for range in category.data %} + + + + + + {% endfor %} + +
RiskThis areaUK average
{{ range.label }}{{ range.value|floatformat }}%{{ range.average|floatformat }}%
{% else %} -

{{ category.data.value }}

- {% if category.data.average %} -

{{ category.data.average }} national average

+ {% if category.data.is_number %} +

{{ category.data.value|floatformat }}{% if category.data.is_percentage %}%{% endif %}

+ {% if category.data.average %} +

{{ category.data.average|floatformat }}{% if category.data.is_percentage %}%{% endif %} national average

+ {% endif %} + {% else %} +

{{ category.data.value }}

+ {% if category.data.average %} +

{{ category.data.average }} national average

+ {% endif %} {% endif %} {% endif %} +
+
+ {% if category.source_url %} +

Data from {{ category.source }}

+ {% else %} +

Data from {{ category.source }}

{% endif %} -
-
- {% if category.source_url %} -

Data from {{ category.source }}

- {% else %} -

Data from {{ category.source }}

- {% endif %} +
- +{% endif %}