Import script updates

This commit covers updates to all of the dataset import scripts, ensuring that their release date, description, and source label are what is expected (according to the 'crowdsourcing' sheet).
mysociety · Nov 15, 2023 · 4890321 · 4890321
1 parent 4035548
commit 4890321
Show file tree

Hide file tree

Showing 41 changed files with 218 additions and 88 deletions.
diff --git a/hub/management/commands/import_aid_alliance_power_postcodes.py b/hub/management/commands/import_aid_alliance_power_postcodes.py
@@ -17,11 +17,12 @@ class Command(BaseAreaImportCommand):
 
     power_postcodes = {
         "label": "Power Postcodes",
-        "description": "Description of Aid Alliance 'Power Postcode' activist groups",
+        "description": "Aid Alliance’s Power Postcodes are activist groups building a stronger connection between communities and MPs in key constituencies, on international development and the UK aid budget.",
         "data_type": "json",
         "category": "movement",
         "subcategory": "groups",
-        "source_label": "Aid Alliance",
+        "release_date": "July 2023",
+        "source_label": "Data from Aid Alliance.",
         "source": "https://aidalliance.org.uk/",
         "source_type": "api",
         "data_url": "https://www.google.com/maps/d/u/0/viewer?mid=15b_tQI0t58rLcBTgFytu2e73jyKrrxFr",
@@ -35,10 +36,11 @@ class Command(BaseAreaImportCommand):
 
     power_postcode_counts = {
         "label": "Number of Power Postcodes",
-        "description": "Number of Aid Alliance 'Power Postcode' activist groups",
+        "description": "Aid Alliance’s Power Postcodes are activist groups building a stronger connection between communities and MPs in key constituencies, on international development and the UK aid budget.",
         "data_type": "integer",
         "category": "movement",
-        "source_label": "Aid Alliance",
+        "release_date": "July 2023",
+        "source_label": "Data from Aid Alliance.",
         "source": "https://aidalliance.org.uk/",
         "source_type": "api",
         "data_url": "https://www.google.com/maps/d/u/0/viewer?mid=15b_tQI0t58rLcBTgFytu2e73jyKrrxFr",

diff --git a/hub/management/commands/import_air_quality_data.py b/hub/management/commands/import_air_quality_data.py
@@ -20,10 +20,11 @@ class Command(BaseCommand):
     message = "Importing constituency air-pollution data"
     defaults = {
         "label": "Air pollution",
-        "description": "Data relating to air pollution, recorded in 2021",
+        "description": "Metrics of pollutants associated with air pollution, aggregated per constituency, recorded automatically across the UK.",
         "data_type": "float",
         "category": "place",
-        "source_label": "Defra",
+        "source_label": "Data from Defra.",
+        "release_date": "2021",
         "is_range": True,
         "source": source_url,
         "source_type": "csv",

diff --git a/hub/management/commands/import_area_age_data.py b/hub/management/commands/import_area_age_data.py
@@ -23,11 +23,12 @@ def handle(self, quiet=False, *args, **options):
 
         defaults = {
             "label": "Constituency age distribution",
-            "description": "Constituency age distribution",
+            "description": "Distribution of the ages of the constituents of each constituency.",
             "data_type": "percent",
             "category": "place",
+            "release_date": "2021",
             "is_range": True,
-            "source_label": "House of Commons Library",
+            "source_label": "Data from ONS (England and Wales), NRS (Scotland), and NISRA (Northern Ireland), collated by House of Commons Library.",
             "source": self.source_url,
             "source_type": "xlxs",
             "table": "areadata",

diff --git a/hub/management/commands/import_area_countries.py b/hub/management/commands/import_area_countries.py
@@ -1,3 +1,5 @@
+from datetime import date
+
 from django.core.management.base import BaseCommand
 
 from tqdm import tqdm
@@ -31,9 +33,10 @@ def create_data_type(self):
             name="country",
             defaults={
                 "data_type": "text",
-                "description": "The country that the constituency is in",
+                "description": "The country that the constituency is in.",
+                "release_date": str(date.today()),
                 "label": "Country of the UK",
-                "source_label": "MapIt",
+                "source_label": "Data from mySociety.",
                 "source": "https://mapit.mysociety.org/",
                 "table": "areadata",
                 "options": options,

diff --git a/hub/management/commands/import_brexit_votes.py b/hub/management/commands/import_brexit_votes.py
@@ -20,8 +20,9 @@ class Command(BaseImportFromDataFrameCommand):
                 "label": "EU (Brexit) leave vote percentage",
                 "description": "Percentage of constituents who voted to leave the EU in the 2016 referendum",
                 "data_type": "percent",
+                "release_date": "February 2017",
                 "category": "opinion",
-                "source_label": "UK Parliament",
+                "source_label": "Data from the House of Commons Library.",
                 "source": "https://commonslibrary.parliament.uk/brexit-votes-by-constituency/",
                 "source_type": "xlxs",
                 "table": "areadata",

diff --git a/hub/management/commands/import_cafod_supporters.py b/hub/management/commands/import_cafod_supporters.py
@@ -26,8 +26,9 @@ class Command(BaseImportFromDataFrameCommand):
     defaults = {
         "data_type": "integer",
         "category": "movement",
-        "source_label": "CAFOD",
+        "source_label": "Data from CAFOD.",
         "source": "https://cafod.org.uk",
+        "release_date": "February 2023",
         "source_type": "google sheet",
         "table": "areadata",
         "exclude_countries": ["Northern Ireland", "Scotland"],
@@ -42,6 +43,7 @@ class Command(BaseImportFromDataFrameCommand):
                 {
                     "subcategory": "supporters_and_activists",
                     "default_value": 10,
+                    "description": "Number of CAFOD activists per constituency."
                 },
             ),
             "col": "activists",
@@ -52,6 +54,7 @@ class Command(BaseImportFromDataFrameCommand):
                 {
                     "subcategory": "places_and_spaces",
                     "default_value": 5,
+                    "description": "Number of CAFOD parishes per constituency."
                 },
             ),
             "col": "parishes",
@@ -62,6 +65,7 @@ class Command(BaseImportFromDataFrameCommand):
                 {
                     "subcategory": "places_and_spaces",
                     "default_value": 5,
+                    "description": "Number of CAFOD schools per constituency."
                 },
             ),
             "col": "schools",

diff --git a/hub/management/commands/import_cen_nzsg_members.py b/hub/management/commands/import_cen_nzsg_members.py
@@ -28,9 +28,10 @@ def create_data_types(self):
             name="cen_member",
             defaults={
                 "data_type": "bool",
-                "description": "Conservative Environment Network membership",
+                "description": "MPs in the CEN’s parliamentary caucus are united around conservative environmentalist principles and championing greater environmental action in Parliament.",
                 "label": "MP membership of Conservative Environment Network (CEN)",
-                "source_label": "CEN, collated by mySociety",
+                "source_label": "Data from CEN, collated by mySociety.",
+                "release_date": "January 2023",
                 "source": "https://www.cen.uk.com/our-caucus",
                 "table": "person__persondata",
                 "options": options,
@@ -41,9 +42,10 @@ def create_data_types(self):
             name="nzsg_member",
             defaults={
                 "data_type": "bool",
-                "description": "Net Zero Scrutiny Group membership",
-                "label": "MP membership of Net Zero Scrutiny Group (NZSG)",
-                "source_label": "collated by DeSmog",
+                "description": "MPs in the Net Zero Scrutiny Group generally oppose the government’s net zero policies.",
+                "label": "MP supporter of Net Zero Scrutiny Group (NZSG)",
+                "source_label": "Data from public sources, collated by DeSmog.",
+                "release_date": "January 2023",
                 "source": "https://www.desmog.com/net-zero-scrutiny-group/",
                 "table": "person__persondata",
                 "options": options,

diff --git a/hub/management/commands/import_child_poverty_data.py b/hub/management/commands/import_child_poverty_data.py
@@ -20,11 +20,12 @@ class Command(BaseImportFromDataFrameCommand):
     data_sets = {
         "constituency_child_poverty": {
             "defaults": {
-                "label": "Child Poverty",
-                "description": "Estimates of percentages of children in poverty",
+                "label": "Estimated child poverty",
+                "description": "Percentage of children living in households with a net income (after housing costs) below 60% of the national median.",
+                "release_date": "March 2022",
                 "data_type": "percent",
                 "category": "place",
-                "source_label": "End Child Poverty",
+                "source_label": "Data from End Child Poverty, based on data from DWP/HMRC.",
                 "source": source_url,
                 "source_type": "xlxs",
                 "data_url": data_url,

diff --git a/hub/management/commands/import_christian_aid_group_locations.py b/hub/management/commands/import_christian_aid_group_locations.py
@@ -18,11 +18,12 @@ class Command(BaseAreaImportCommand):
     data_file = settings.BASE_DIR / "data" / "christian_aid_groups.csv"
     defaults = {
         "label": "Christian Aid groups",
-        "description": "Descriptions of Christian Aid Groups",
+        "description": "Descriptions of Christian Aid groups",
         "data_type": "json",
         "category": "movement",
         "subcategory": "groups",
-        "source_label": "Christian Aid",
+        "release_date": "April 2023",
+        "source_label": "Data from Christian Aid.",
         "source": "",
         "source_type": "api",
         "data_url": "",
@@ -34,10 +35,11 @@ class Command(BaseAreaImportCommand):
 
     count_defaults = {
         "label": "Number of Christian Aid groups",
-        "description": "Number of Christian Aid Groups",
+        "description": "Number of Christian Aid groups",
         "data_type": "integer",
+        "release_date": "April 2023",
         "category": "movement",
-        "source_label": "Christian Aid",
+        "source_label": "Data from Christian Aid.",
         "source": "",
         "source_type": "api",
         "data_url": "",

diff --git a/hub/management/commands/import_cost_of_living_mrp_data.py b/hub/management/commands/import_cost_of_living_mrp_data.py
@@ -17,7 +17,8 @@ class Command(BaseImportFromDataFrameCommand):
         "data_type": "percent",
         "category": "opinion",
         "subcategory": "cost_of_living",
-        "source_label": "Survation 2023 MRP polling for 38Degrees",
+        "release_date": "March 2023",
+        "source_label": "Survation 2023 MRP polling, commissioned by 38Degrees.",
         "source": "https://home.38degrees.org.uk/2023/03/08/the-true-cost-of-this-crisis/",
         "source_type": "google sheet",
         "table": "areadata",
@@ -42,6 +43,18 @@ class Command(BaseImportFromDataFrameCommand):
         "mental_health_cost_of_living": "Mental health has been impacted by the cost of living crisis",
     }
 
+    descriptions = {
+        "not_able_to_afford_mortgage_rent": "Estimated percentage of constituents worried about not being able to afford their mortgage or rent in the next year.",
+        "not_able_to_pay_energy_bills": "Estimated percentage of constituents worried about not being able to pay their energy bills in the next year.",
+        "worried_foodbank": "Estimated percentage of constituents worried about having to use a foodbank in the next year.",
+        "conservatives_dont_understand_impact": "Estimated percentage of constituents who think that Rishi Sunak and the Conservative Government do not understand the impact the cost of living crisis is having on people.",
+        "missed_bill_payment": "Estimated percentage of constituents that have missed a bill payment in the last six months.",
+        "missed_rent_payment": "Estimated percentage of constituents that have missed a rent payment in the last six months.",
+        "missed_credit_payment": "Estimated percentage of constituents that have missed a credit payment in the last six months.",
+        "cant_afford_heating": "Estimated percentage of constituents that have not been able to afford to turn the heating on at home when they have felt cold in the past month.",
+        "mental_health_cost_of_living": "Estimated percentage of constituents whose mental health has been impacted by the cost of living crisis.",
+    }
+
     def add_data_sets(self, df):
         for col in df.columns:
             if col != "gss_code":
@@ -50,6 +63,7 @@ def add_data_sets(self, df):
                     "col": col,
                 }
                 self.data_sets[col]["defaults"]["label"] = self.labels[col]
+                self.data_sets[col]["defaults"]["description"] = self.descriptions[col]
 
         super().add_data_sets()
 

diff --git a/hub/management/commands/import_efpc_fuel_poverty_data.py b/hub/management/commands/import_efpc_fuel_poverty_data.py
@@ -19,11 +19,12 @@ class Command(BaseImportFromDataFrameCommand):
     data_sets = {
         "efpc_constituency_fuel_poverty": {
             "defaults": {
-                "label": "Households in fuel poverty",
-                "description": "Percentage of households in fuel poverty",
+                "label": "Estimated households in fuel poverty",
+                "description": "Percentage of households in fuel poverty, including adjustment for Energy Price Guarantee, using National Energy Action base number of households in fuel poverty.",
                 "data_type": "percent",
+                "release_date": "July 2023",
                 "category": "place",
-                "source_label": "End Fuel Poverty Coalition",
+                "source_label": "Data from End Fuel Poverty Coalition.",
                 "source": source_url,
                 "source_type": "xlxs",
                 "data_url": "",

diff --git a/hub/management/commands/import_ethnicity_data.py b/hub/management/commands/import_ethnicity_data.py
@@ -20,7 +20,8 @@ class Command(BaseCommand):
         "description": "Ethnicity as of the 2021 census (England and Wales)",
         "data_type": "percent",
         "category": "place",
-        "source_label": "House of Commons Library",
+        "release_date": "2021",
+        "source_label": "Data From House of Commons Library",
         "is_range": True,
         "source": source_url,
         "source_type": "xlxs",

diff --git a/hub/management/commands/import_flood_risk_data.py b/hub/management/commands/import_flood_risk_data.py
@@ -14,10 +14,11 @@ class Command(BaseCommand):
     message = "Importing constituency flood risk data"
     defaults = {
         "label": "Flood risk from rivers or sea",
-        "description": "Data relating to risk of flooding from rivers or sea, recorded in 2018",
+        "description": "Data relating to risk of flooding from rivers or sea across the UK, aggregated by constituency.",
+        "release_date": "2018",
         "data_type": "percent",
         "category": "place",
-        "source_label": "Defra",
+        "source_label": "Data from Defra.",
         "is_range": True,
         "source_type": "csv",
         "table": "areadata",

diff --git a/hub/management/commands/import_foe_groups.py b/hub/management/commands/import_foe_groups.py
@@ -20,8 +20,9 @@ class Command(BaseLatLongImportCommand):
         "category": "movement",
         "subcategory": "groups",
         "label": "Active Friends of the Earth groups",
-        "description": "Active Friends of the Earth groups by constituency",
-        "source_label": "Friends of the Earth",
+        "description": "Descriptions of active Friends of the Earth groups by constituency.",
+        "source_label": "Data from Friends of the Earth.",
+        "release_date": "November 2022",
         "source": "https://friendsoftheearth.uk/",
         "source_type": "google sheet",
         "table": "areadata",

diff --git a/hub/management/commands/import_foe_groups_count.py b/hub/management/commands/import_foe_groups_count.py
@@ -19,7 +19,9 @@ class Command(BaseImportFromDataFrameCommand):
         "data_type": "integer",
         "category": "movement",
         "subcategory": "groups",
-        "source_label": "Friends of the Earth",
+        "description": "Number of Friends of the Earth groups per constituency.",
+        "release_date": "Novemeber 2022",
+        "source_label": "Data from Friends of the Earth.",
         "source": "https://friendsoftheearth.uk/",
         "source_type": "google sheet",
         "table": "areadata",

diff --git a/hub/management/commands/import_foe_supporters.py b/hub/management/commands/import_foe_supporters.py
@@ -19,7 +19,8 @@ class Command(BaseImportFromDataFrameCommand):
         "data_type": "integer",
         "category": "movement",
         "subcategory": "supporters_and_activists",
-        "source_label": "Friends of the Earth",
+        "release_date": "November 2022",
+        "source_label": "Data from Friends of the Earth.",
         "source": "https://friendsoftheearth.uk/",
         "source_type": "google sheet",
         "table": "areadata",
@@ -30,11 +31,11 @@ class Command(BaseImportFromDataFrameCommand):
 
     data_sets = {
         "constituency_foe_activists_count": {
-            "defaults": defaults,
+            "defaults": defaults.extend({"description": "Number of Friends of the Earth activists per constituency."}),
             "col": "activists",
         },
         "constituency_foe_supporters_count": {
-            "defaults": defaults,
+            "defaults": defaults.extend({"description": "Number of Friends of the Earth supporters per constituency."}),
             "col": "supporters",
         },
     }

diff --git a/hub/management/commands/import_foodbank_count.py b/hub/management/commands/import_foodbank_count.py
@@ -15,10 +15,11 @@ class Command(BaseConstituencyCountImportCommand):
 
     defaults = {
         "label": "Number of Trussell Trust foodbanks",
-        "description": "Number of Trussell Trust foodbanks",
+        "description": "Number of Trussell Trust foodbanks per constituency.",
         "data_type": "integer",
         "category": "movement",
-        "source_label": "Trussell Trust",
+        "release_date": "December 2022",
+        "source_label": "Data from Trussell Trust.",
         "source": "https://www.trusselltrust.org/",
         "source_type": "csv",
         "data_url": "",

diff --git a/hub/management/commands/import_gbgw_events.py b/hub/management/commands/import_gbgw_events.py
@@ -28,11 +28,12 @@ def add_data_sets(self, df):
             name="constituency_gbgw_2022_event_count",
             defaults={
                 "label": "Number of Great Big Green Week 2022 events",
-                "description": "Number of Great Big Green Week 2022 events",
+                "description": "Number of Great Big Green Week 2022 events per constituency.",
+                "release_date": "October 2022",
                 "data_type": "integer",
                 "category": "movement",
                 "subcategory": "events",
-                "source_label": "The Climate Coalition",
+                "source_label": "Data from The Climate Coalition.",
                 "source": "https://greatbiggreenweek.com/",
                 "source_type": "google sheet",
                 "data_url": "",
@@ -48,7 +49,7 @@ def add_data_sets(self, df):
             defaults={
                 "data_type": "integer",
                 "label": "Number of Great Big Green Week 2022 events",
-                "description": "Number of Great Big Green Week 2022 events",
+                "description": "Number of Great Big Green Week 2022 events per constituency.",
             },
         )
 

diff --git a/hub/management/commands/import_hnh_polling_data.py b/hub/management/commands/import_hnh_polling_data.py
@@ -33,7 +33,8 @@ class Command(BaseImportFromDataFrameCommand):
         "data_type": "percent",
         "category": "opinion",
         "is_range": True,
-        "source_label": "Focaldata 2023 MRP polling for HOPE not hate",
+        "release_date": "April 2023",
+        "source_label": "Focaldata 2023 MRP polling, commissioned by HOPE not hate.",
         "source": "https://hopenothate.org.uk/research/",
         "source_type": "google sheet",
         "table": "areadata",

diff --git a/hub/management/commands/import_imd_data.py b/hub/management/commands/import_imd_data.py
@@ -1,3 +1,5 @@
+from datetime import date
+
 import pandas as pd
 from mysoc_dataset import get_dataset_url
 
@@ -15,10 +17,11 @@ class Command(BaseImportFromDataFrameCommand):
         "constituency_imd": {
             "defaults": {
                 "source": "https://mysociety.github.io/composite_uk_imd/",
-                "source_label": "mySociety",
+                "source_label": "Data from ONS (England and Wales), NRS (Scotland), and NISRA (Northern Ireland), collated and standardised by mySociety.",
                 "name": "constituency_imd",
-                "description": "Index of Multiple Deprivation",
+                "description": "Deciles of deprivation, from 1 (most deprived) to 10 (least deprived). This uses a composite measure of deprivation (including income, employment, education, skills, health, crime, and housing) standardised across the countries of the UK.",
                 "label": "Index of Multiple Deprivation (IMD)",
+                "release_date": str(date.today()),
                 "data_type": "integer",
                 "category": "place",
                 "source_type": "csv",