Skip to content

Commit

Permalink
fixes to council import script
Browse files Browse the repository at this point in the history
  • Loading branch information
struan committed Nov 11, 2024
1 parent b1f8f3f commit f036de9
Show file tree
Hide file tree
Showing 6 changed files with 91 additions and 23 deletions.
60 changes: 46 additions & 14 deletions crowdsourcer/management/commands/import_councils.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,15 @@
class Command(BaseCommand):
help = "import councils"

council_file = settings.BASE_DIR / "data" / "merged_contacts.csv"
council_file = (
settings.BASE_DIR / "data" / "scorecards-2025" / "council_contacts.csv"
)

council_map = {
"Cheshire West & Chester Council": "Cheshire West and Chester Council",
"Cumbria Council": "Cumbria County Council",
"East Sussex Council": "East Sussex County Council",
}

def add_arguments(self, parser):
parser.add_argument(
Expand All @@ -41,33 +49,50 @@ def add_arguments(self, parser):

parser.add_argument("--council_list", help="file to import data from")

def get_council(self, row):
if row.get("gssNumber"):
try:
council = PublicAuthority.objects.get(unique_id=row["gssNumber"])
return council
except PublicAuthority.DoesNotExist:
print("no gss found")
return None

if row.get("council"):
try:
council = PublicAuthority.objects.get(
name=self.council_map.get(row["council"], row["council"])
)
return council
except PublicAuthority.DoesNotExist:
print(f"no name found: {row['council']}")
return None

return None

def handle(self, quiet: bool = False, session: str = None, *args, **options):
if options.get("council_list") is not None:
self.council_file = options["council_list"]

df = pd.read_csv(
self.council_file,
usecols=[
"firstName",
"surname",
"name",
"council",
"gssNumber",
"email",
],
)

session = MarkingSession.objects.get(label=session)
rt = ResponseType.objects.get(type="Right of Reply")
for index, row in df.iterrows():
if pd.isna(row["email"]) or pd.isna(row["gssNumber"]):
if pd.isna(row["email"]):
continue

try:
council = PublicAuthority.objects.get(unique_id=row["gssNumber"])
except PublicAuthority.DoesNotExist:
self.stdout.write(
f"No council with GSS of {row['gssNumber']}, {row['council']} found"
)
council = self.get_council(row)

if not council:
self.stdout.write(f"No council {row['council']} found")
continue

if council.do_not_mark:
Expand All @@ -90,7 +115,12 @@ def handle(self, quiet: bool = False, session: str = None, *args, **options):

if User.objects.filter(username=row["email"]).exists():
u = User.objects.get(username=row["email"])
if (
if not hasattr(u, "marker"):
m = Marker.objects.create(
user=u, authority=council, response_type=rt
)
m.marking_session.set([session])
elif (
u.marker.authority == council
and not u.marker.marking_session.filter(pk=session.pk).exists()
):
Expand Down Expand Up @@ -121,6 +151,7 @@ def handle(self, quiet: bool = False, session: str = None, *args, **options):
a, _ = Assigned.objects.update_or_create(
user=u,
authority=c,
response_type=rt,
marking_session=session,
)
u.marker.authority = None
Expand All @@ -131,13 +162,14 @@ def handle(self, quiet: bool = False, session: str = None, *args, **options):
self.stdout.write(f"user already exists for email: {row['email']}")
continue

firstname, surname = row["name"].split(" ", maxsplit=1)
if options["add_users"] is True:
u, created = User.objects.update_or_create(
username=row["email"],
defaults={
"email": row["email"],
"first_name": row["firstName"],
"last_name": row["surname"],
"first_name": firstname,
"last_name": surname,
},
)
u.save()
Expand Down
5 changes: 5 additions & 0 deletions crowdsourcer/tests/data/council_new_joint_users_2025.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
,name,councilShortName,council,gssNumber,email,official-name
0,Armagh Staff,"Armagh, Banbridge and Craigavon","Armagh City, Banbridge and Craigavon Borough Council",N09000002,[email protected],"Armagh City, Banbridge and Craigavon Borough Council"
1,Aberdeen Staff,Aberdeenshire,Aberdeenshire Council,S12000034,[email protected],Aberdeenshire Council
2,Aberdeen Staff,Aberdeen City,Aberdeen City Council,S12000033,[email protected],Aberdeen City Council
3,Adur Staff,Adur,Adur District Council,E07000223,[email protected],Adur District Council
5 changes: 5 additions & 0 deletions crowdsourcer/tests/data/council_new_users_2025.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
,name,councilShortName,council,gssNumber,email,official-name
0,Armagh Staff,"Armagh, Banbridge and Craigavon","Armagh City, Banbridge and Craigavon Borough Council",N09000002,[email protected],"Armagh City, Banbridge and Craigavon Borough Council"
1,Aberdeen Staff,Aberdeenshire,Aberdeenshire Council,S12000034,[email protected],Aberdeenshire Council
2,Aberdeen Staff,Aberdeen City,Aberdeen City Council,S12000033,[email protected],Aberdeen City Council
3,Adur Staff,Adur,Adur District Council,E07000223,[email protected],Adur District Council
5 changes: 5 additions & 0 deletions crowdsourcer/tests/data/council_split_users_2025.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
,name,councilShortName,council,gssNumberOld,email,official-name
0,Armagh Staff,"Armagh, Banbridge and Craigavon","Armagh City, Banbridge and Craigavon Borough Council",N09000002,[email protected],"Armagh City, Banbridge and Craigavon Borough Council"
1,Aberdeen Staff,Aberdeenshire,Aberdeenshire Council,S12000034,[email protected],Aberdeenshire Council
2,Aberdeen Staff,Aberdeen City,Aberdeen City Council,S12000033,[email protected],Aberdeen City Council
3,Adur Staff,Adur,Adur District Council,E07000223,[email protected],Adur District Council
5 changes: 5 additions & 0 deletions crowdsourcer/tests/data/merged_contacts_2025.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
,name,councilShortName,council,gssNumberOld,email,official-name
0,Armagh Staff,"Armagh, Banbridge and Craigavon","Armagh City, Banbridge and Craigavon Borough Council",N09000002,[email protected],"Armagh City, Banbridge and Craigavon Borough Council"
1,Aberdeen Staff,Aberdeenshire,Aberdeenshire Council,S12000034,[email protected],Aberdeenshire Council
2,Aberdeen Staff,Aberdeen City,Aberdeen City Council,S12000033,[email protected],Aberdeen City Council
3,Adur Staff,Adur,Adur District Council,E07000223,[email protected],Adur District Council
34 changes: 25 additions & 9 deletions crowdsourcer/tests/test_commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,9 @@ class ImportCouncilsTestCase(BaseCommandTestCase):

def test_import_councils_no_commit(self):
data_file = (
pathlib.Path(__file__).parent.resolve() / "data" / "merged_contacts.csv"
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "merged_contacts_2025.csv"
)

self.call_command("import_councils", session="Default", council_list=data_file)
Expand All @@ -120,7 +122,9 @@ def test_import_councils_no_commit(self):

def test_import_councils(self):
data_file = (
pathlib.Path(__file__).parent.resolve() / "data" / "merged_contacts.csv"
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "merged_contacts_2025.csv"
)

self.call_command(
Expand Down Expand Up @@ -156,7 +160,9 @@ def test_import_councils(self):

def test_import_for_new_session_all_same_users(self):
data_file = (
pathlib.Path(__file__).parent.resolve() / "data" / "merged_contacts.csv"
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "merged_contacts_2025.csv"
)

self.call_command(
Expand Down Expand Up @@ -210,7 +216,9 @@ def test_import_for_new_session_all_same_users(self):

def test_import_for_new_session_new_users(self):
data_file = (
pathlib.Path(__file__).parent.resolve() / "data" / "merged_contacts.csv"
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "merged_contacts_2025.csv"
)

self.call_command(
Expand All @@ -226,7 +234,9 @@ def test_import_for_new_session_new_users(self):
2, Marker.objects.filter(marking_session__label="Default").count()
)
data_file = (
pathlib.Path(__file__).parent.resolve() / "data" / "council_new_users.csv"
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "council_new_users_2025.csv"
)

self.call_command(
Expand Down Expand Up @@ -288,7 +298,9 @@ def test_import_for_new_session_new_users(self):

def test_import_for_new_session_council_split_users(self):
data_file = (
pathlib.Path(__file__).parent.resolve() / "data" / "merged_contacts.csv"
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "merged_contacts_2025.csv"
)

self.call_command(
Expand All @@ -304,7 +316,9 @@ def test_import_for_new_session_council_split_users(self):
2, Marker.objects.filter(marking_session__label="Default").count()
)
data_file = (
pathlib.Path(__file__).parent.resolve() / "data" / "council_split_users.csv"
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "council_split_users_2025.csv"
)

self.call_command(
Expand All @@ -329,7 +343,9 @@ def test_import_for_new_session_council_split_users(self):

def test_import_for_new_session_council_new_joint_users(self):
data_file = (
pathlib.Path(__file__).parent.resolve() / "data" / "merged_contacts.csv"
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "merged_contacts_2025.csv"
)

self.call_command(
Expand All @@ -347,7 +363,7 @@ def test_import_for_new_session_council_new_joint_users(self):
data_file = (
pathlib.Path(__file__).parent.resolve()
/ "data"
/ "council_new_joint_users.csv"
/ "council_new_joint_users_2025.csv"
)

self.call_command(
Expand Down

0 comments on commit f036de9

Please sign in to comment.