From 282ab31a5200938d033b7842218f7fe27e3a7f1c Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Fri, 10 Jan 2025 11:35:40 -0800 Subject: [PATCH 01/11] wip --- .../Controllers/Ballot/castVoteController.ts | 6 +++ packages/backend/src/Routes/ballot.routes.ts | 7 +-- .../ElectionForm/CreateElectionDialog.tsx | 4 +- .../src/components/UploadElections.tsx | 23 ++++----- .../frontend/src/components/cvrParsers.tsx | 12 +++++ packages/frontend/src/components/util.tsx | 3 +- packages/shared/src/domain_model/Ballot.ts | 4 ++ .../shared/src/domain_model/permissions.ts | 47 ++++++++++--------- 8 files changed, 60 insertions(+), 46 deletions(-) create mode 100644 packages/frontend/src/components/cvrParsers.tsx diff --git a/packages/backend/src/Controllers/Ballot/castVoteController.ts b/packages/backend/src/Controllers/Ballot/castVoteController.ts index e3280e7d..ddaf9747 100644 --- a/packages/backend/src/Controllers/Ballot/castVoteController.ts +++ b/packages/backend/src/Controllers/Ballot/castVoteController.ts @@ -15,6 +15,8 @@ import { IElectionRequest } from "../../IRequest"; import { Response, NextFunction } from 'express'; import { io } from "../../socketHandler"; import { Server } from "socket.io"; +import { expectPermission } from "../controllerUtils"; +import { permissions } from "@equal-vote/star-vote-shared/domain_model/permissions"; const ElectionsModel = ServiceLocator.electionsDb(); const ElectionRollModel = ServiceLocator.electionRollDb(); @@ -101,6 +103,10 @@ async function makeBallotEvent(req: IElectionRequest, targetElection: Election, async function uploadBallotsController(req: IElectionRequest, res: Response, next: NextFunction) { Logger.info(req, "Upload Ballots Controller"); + expectPermission(req.user_auth.roles, permissions.canUploadBallots); + + //TODO: if it's a public_archive item, also check canUpdatePublicArchive instead + const targetElection = req.election; if (targetElection == null){ const errMsg = "Invalid Ballot: invalid election Id"; diff --git a/packages/backend/src/Routes/ballot.routes.ts b/packages/backend/src/Routes/ballot.routes.ts index 404ed4cf..c2665418 100644 --- a/packages/backend/src/Routes/ballot.routes.ts +++ b/packages/backend/src/Routes/ballot.routes.ts @@ -232,12 +232,7 @@ ballotRouter.post('/Election/:id/vote', asyncHandler(castVoteController)) * type: array * items: * type: object - * properties: - * ballot: - * type: object - * $ref: '#/components/schemas/NewBallot' - * voter_id: - * type: string + * $ref: '#/components/schemas/NewBallotWithVoterID' * respon ses: * 200: * description: All Ballots Processed diff --git a/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx b/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx index 15bb1da1..4e002b08 100644 --- a/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx +++ b/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx @@ -310,8 +310,8 @@ export default () => { }}> setElection({...election, settings: { diff --git a/packages/frontend/src/components/UploadElections.tsx b/packages/frontend/src/components/UploadElections.tsx index 26305c96..41b083d2 100644 --- a/packages/frontend/src/components/UploadElections.tsx +++ b/packages/frontend/src/components/UploadElections.tsx @@ -3,13 +3,21 @@ import { Box, Button, Checkbox, FormControlLabel, FormGroup, MenuItem, Paper, Se import { useRef, useState } from "react"; import { useSubstitutedTranslation } from "./util"; import EnhancedTable from "./EnhancedTable"; +import { rankColumnCSV } from "./cvrParsers"; export default () => { - const [votingMethod, setVotingMethod] = useState('IRV') const [addToPublicArchive, setAddToPublicArchive] = useState(false) const [cvrs, setCvrs] = useState([]) const {t} = useSubstitutedTranslation(); const inputRef = useRef(null) + const [electionsSubmitted, setElectionsSubmitted] = useState(false); + + const submitElections = () => { + setElectionsSubmitted(true) + + + fetch(cvrs[0].url).then(res => res.text()).then(s => rankColumnCSV(s)) + } const handleDragOver = (e) => { e.preventDefault() @@ -42,17 +50,6 @@ export default () => { gap={2} > Upload Election(s) - {/* TODO: add a sys admin permission check*/ } @@ -114,6 +111,6 @@ export default () => { emptyContent={

No files selected

} /> - + } \ No newline at end of file diff --git a/packages/frontend/src/components/cvrParsers.tsx b/packages/frontend/src/components/cvrParsers.tsx new file mode 100644 index 00000000..d4acfa18 --- /dev/null +++ b/packages/frontend/src/components/cvrParsers.tsx @@ -0,0 +1,12 @@ +import { NewBallotWithVoterID } from "@equal-vote/star-vote-shared/domain_model/Ballot"; +import * as CSV from 'csv-string'; + +const parseCSV = (text : string) => { + console.log(CSV.parse(text)); +} + +// ported from https://github.com/fairvotereform/rcv_cruncher/blob/9bb9f8482290033ff7b31d6b091186474e7afff6/src/rcv_cruncher/parsers.py +export const rankColumnCSV = (text: string) : NewBallotWithVoterID[] => { + parseCSV(text); + return []; +} \ No newline at end of file diff --git a/packages/frontend/src/components/util.tsx b/packages/frontend/src/components/util.tsx index fe5ce33e..257d331d 100644 --- a/packages/frontend/src/components/util.tsx +++ b/packages/frontend/src/components/util.tsx @@ -40,7 +40,6 @@ export const methodValueToTextKey = { }; export const MailTo = ({ children }) => { - const { t } = useSubstitutedTranslation(); const { setSnack } = useSnackbar(); // https://adamsilver.io/blog/the-trouble-with-mailto-email-links-and-what-to-do-instead/ return @@ -155,7 +154,7 @@ export const useSubstitutedTranslation = (electionTermType = 'election', v = {}) if (i % 3 == 0) return str; if (i % 3 == 2) return ''; if (parts[i + 1].startsWith('mailto')) { - return {parts[i]} + return {parts[i]} } else { return {parts[i]} } diff --git a/packages/shared/src/domain_model/Ballot.ts b/packages/shared/src/domain_model/Ballot.ts index 67677971..f2b2539e 100644 --- a/packages/shared/src/domain_model/Ballot.ts +++ b/packages/shared/src/domain_model/Ballot.ts @@ -4,6 +4,10 @@ import { Race } from "./Race"; import { Uid } from "./Uid"; import { Vote } from "./Vote"; +export interface NewBallotWithVoterID { + voter_id: string; + ballot: NewBallot; +} export interface Ballot { ballot_id: Uid; //ID if ballot election_id: Uid; //ID of election ballot is cast in diff --git a/packages/shared/src/domain_model/permissions.ts b/packages/shared/src/domain_model/permissions.ts index 29b8c7ca..8aae4858 100644 --- a/packages/shared/src/domain_model/permissions.ts +++ b/packages/shared/src/domain_model/permissions.ts @@ -3,29 +3,30 @@ import { roles} from "./roles" export type permission = roles[] export const permissions = { - canEditElectionRoles: [roles.system_admin, roles.owner], - canViewElection: [roles.system_admin, roles.owner, roles.admin, roles.auditor, roles.credentialer], - canEditElection: [roles.system_admin, roles.owner, roles.admin], - canDeleteElection: [roles.system_admin, roles.owner], - canEditElectionRoll: [roles.system_admin, roles.owner], - canAddToElectionRoll: [roles.system_admin, roles.owner, roles.admin], - canViewElectionRoll: [roles.system_admin, roles.owner, roles.admin, roles.auditor, roles.credentialer], - canFlagElectionRoll: [roles.system_admin, roles.owner, roles.admin, roles.auditor, roles.credentialer], - canApproveElectionRoll: [roles.system_admin, roles.owner, roles.admin, roles.credentialer], - canUnflagElectionRoll: [roles.system_admin, roles.owner, roles.admin], - canInvalidateElectionRoll:[roles.system_admin, roles.owner, roles.admin], - canDeleteElectionRoll: [roles.system_admin, roles.owner], - canViewElectionRollIDs: [roles.system_admin, roles.auditor], - canViewBallots: [roles.system_admin, roles.owner, roles.admin, roles.auditor], - canDeleteAllBallots: [roles.system_admin, roles.owner, roles.admin], - canViewBallot: [roles.system_admin], - canEditBallot: [roles.system_admin, roles.owner], - canFlagBallot: [roles.system_admin, roles.owner, roles.admin, roles.auditor], - canInvalidateBallot: [roles.system_admin, roles.owner], - canEditElectionState: [roles.system_admin, roles.owner], - canViewPreliminaryResults:[roles.system_admin, roles.owner, roles.admin, roles.auditor], - canSendEmails: [roles.system_admin, roles.owner, roles.admin], - canUpdatePublicArchive: [roles.system_admin], + canEditElectionRoles: [roles.system_admin, roles.owner], + canViewElection: [roles.system_admin, roles.owner, roles.admin, roles.auditor, roles.credentialer], + canEditElection: [roles.system_admin, roles.owner, roles.admin], + canDeleteElection: [roles.system_admin, roles.owner], + canEditElectionRoll: [roles.system_admin, roles.owner], + canAddToElectionRoll: [roles.system_admin, roles.owner, roles.admin], + canViewElectionRoll: [roles.system_admin, roles.owner, roles.admin, roles.auditor, roles.credentialer], + canFlagElectionRoll: [roles.system_admin, roles.owner, roles.admin, roles.auditor, roles.credentialer], + canApproveElectionRoll: [roles.system_admin, roles.owner, roles.admin, roles.credentialer], + canUnflagElectionRoll: [roles.system_admin, roles.owner, roles.admin], + canInvalidateElectionRoll: [roles.system_admin, roles.owner, roles.admin], + canDeleteElectionRoll: [roles.system_admin, roles.owner], + canViewElectionRollIDs: [roles.system_admin, roles.auditor], + canViewBallots: [roles.system_admin, roles.owner, roles.admin, roles.auditor], + canDeleteAllBallots: [roles.system_admin, roles.owner, roles.admin], + canViewBallot: [roles.system_admin], + canEditBallot: [roles.system_admin, roles.owner], + canFlagBallot: [roles.system_admin, roles.owner, roles.admin, roles.auditor], + canInvalidateBallot: [roles.system_admin, roles.owner], + canEditElectionState: [roles.system_admin, roles.owner], + canViewPreliminaryResults: [roles.system_admin, roles.owner, roles.admin, roles.auditor], + canSendEmails: [roles.system_admin, roles.owner, roles.admin], + canUpdatePublicArchive: [roles.system_admin], + canUploadBallots: [roles.system_admin, roles.owner], } export const hasPermission = (roles:roles[],permission:permission) => { From 870ce60c939eca150a9e7a549bafb7c3f398f4cf Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Sat, 11 Jan 2025 11:24:34 -0800 Subject: [PATCH 02/11] Create elections from CSV --- package-lock.json | 12 +++ packages/frontend/package.json | 1 + .../ElectionForm/CreateElectionDialog.tsx | 2 +- .../src/components/UploadElections.tsx | 81 ++++++++++++++++++- .../frontend/src/components/cvrParsers.tsx | 59 ++++++++++++-- 5 files changed, 145 insertions(+), 10 deletions(-) diff --git a/package-lock.json b/package-lock.json index 33106c1a..96f5ebdf 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9408,6 +9408,11 @@ "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" }, + "node_modules/papaparse": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.5.1.tgz", + "integrity": "sha512-EuEKUhyxrHVozD7g3/ztsJn6qaKse8RPfR6buNB2dMJvdtXNhcw8jccVi/LxNEY3HVrV6GO6Z4OoeCG9Iy9wpA==" + }, "node_modules/parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", @@ -12081,6 +12086,7 @@ "js-yaml": "^4.1.0", "jwt-decode": "^3.1.2", "luxon": "^3.3.0", + "papaparse": "^5.5.1", "react": "^17.0.2", "react-csv": "^2.2.2", "react-dom": "^17.0.2", @@ -14101,6 +14107,7 @@ "js-yaml": "^4.1.0", "jwt-decode": "^3.1.2", "luxon": "^3.3.0", + "papaparse": "^5.5.1", "react": "^17.0.2", "react-csv": "^2.2.2", "react-dom": "^17.0.2", @@ -19793,6 +19800,11 @@ "resolved": "https://registry.npmjs.org/packet-reader/-/packet-reader-1.0.0.tgz", "integrity": "sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==" }, + "papaparse": { + "version": "5.5.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.5.1.tgz", + "integrity": "sha512-EuEKUhyxrHVozD7g3/ztsJn6qaKse8RPfR6buNB2dMJvdtXNhcw8jccVi/LxNEY3HVrV6GO6Z4OoeCG9Iy9wpA==" + }, "parent-module": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/parent-module/-/parent-module-1.0.1.tgz", diff --git a/packages/frontend/package.json b/packages/frontend/package.json index 4b116451..ffc7be55 100644 --- a/packages/frontend/package.json +++ b/packages/frontend/package.json @@ -18,6 +18,7 @@ "js-yaml": "^4.1.0", "jwt-decode": "^3.1.2", "luxon": "^3.3.0", + "papaparse": "^5.5.1", "react": "^17.0.2", "react-csv": "^2.2.2", "react-dom": "^17.0.2", diff --git a/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx b/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx index 4e002b08..14f8874b 100644 --- a/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx +++ b/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx @@ -49,7 +49,7 @@ export const CreateElectionContextProvider = ({children}) => { /////// DIALOG ///// -const defaultElection: NewElection = { +export const defaultElection: NewElection = { title: '', owner_id: '', description: '', diff --git a/packages/frontend/src/components/UploadElections.tsx b/packages/frontend/src/components/UploadElections.tsx index 41b083d2..13f7a4c9 100644 --- a/packages/frontend/src/components/UploadElections.tsx +++ b/packages/frontend/src/components/UploadElections.tsx @@ -4,6 +4,12 @@ import { useRef, useState } from "react"; import { useSubstitutedTranslation } from "./util"; import EnhancedTable from "./EnhancedTable"; import { rankColumnCSV } from "./cvrParsers"; +import { v4 as uuidv4 } from 'uuid'; +import Papa from 'papaparse'; +import { usePostElection } from "~/hooks/useAPI"; +import useAuthSession from "./AuthSessionContextProvider"; +import { defaultElection } from "./ElectionForm/CreateElectionDialog"; +import { Candidate } from "@equal-vote/star-vote-shared/domain_model/Candidate"; export default () => { const [addToPublicArchive, setAddToPublicArchive] = useState(false) @@ -11,12 +17,83 @@ export default () => { const {t} = useSubstitutedTranslation(); const inputRef = useRef(null) const [electionsSubmitted, setElectionsSubmitted] = useState(false); + const { error: postError, isPending, makeRequest: postElection } = usePostElection() + const authSession = useAuthSession() + const submitElections = () => { setElectionsSubmitted(true) + cvrs.forEach(cvr => { + // #1: Parse CSV + const post_process = async (parsed_csv) => { + // #2 : Infer Election Settings + const errorRows = new Set(parsed_csv.errors.map(error => error.row)) + const rankFields = parsed_csv.meta.fields.filter((field:string) => field.startsWith('rank')); + const maxRankings = rankFields.length; + let candidateNames = new Set(); + parsed_csv.data.forEach((row, i) => { + if(errorRows.has(i)) return; + candidateNames = candidateNames.union(new Set(rankFields.map(rankField => row[rankField]))) + }) + candidateNames.delete('skipped'); + candidateNames.delete('overvote'); + // TODO: infer num winners + + // #3 : Create (or fetch) Election + const election = await postElection({ + Election: { + ...defaultElection, + title: cvr.name.split('.')[0], + state: 'closed', + owner_id: authSession.getIdField('sub'), + settings: { + ...defaultElection.settings, + max_rankings: maxRankings + }, + races: [ + { + race_id: uuidv4(), + voting_method: 'IRV', + title: cvr.name.split('.')[0], + candidates: [...candidateNames].map(name => ({ + candidate_id: uuidv4(), + candidate_name: name + })) as Candidate[], + num_winners: 1 + } + ] + }, + }) - fetch(cvrs[0].url).then(res => res.text()).then(s => rankColumnCSV(s)) + if (!election){ + parsed_csv.errors.push({ + code: "ElectionCreationFailed", + message: postError, + row: -1, + type: "ElectionCreationFailed" + }) + return; + }; + + //// #4 : Convert Rows to Ballots + //rankColumnCSV(results, 'election_id') + // TODO: store results.errors somewhere + + } + Papa.parse(cvr.url, { + header: true, + download: true, + dynamicTyping: true, + complete: post_process + }) + // infer election settings from name, and csv headers + // create election (w/ first race) + // convert rows to ballots + // upload ballots + + // optional: add more election settings based on ballots (ex. max rankings) + }) } const handleDragOver = (e) => { @@ -111,6 +188,6 @@ export default () => { emptyContent={

No files selected

} /> - + } \ No newline at end of file diff --git a/packages/frontend/src/components/cvrParsers.tsx b/packages/frontend/src/components/cvrParsers.tsx index d4acfa18..e2250cef 100644 --- a/packages/frontend/src/components/cvrParsers.tsx +++ b/packages/frontend/src/components/cvrParsers.tsx @@ -1,12 +1,57 @@ import { NewBallotWithVoterID } from "@equal-vote/star-vote-shared/domain_model/Ballot"; -import * as CSV from 'csv-string'; -const parseCSV = (text : string) => { - console.log(CSV.parse(text)); -} +/* + Example Input (assuming papa parse) + + { + data: [ + {ballotID: 7, ward: 400, rank1: 'Terry Seamens', rank2: 'Terry Seamens'} , + {ballotID: 8, ward: 400, rank1: 'Terry Seamens', rank2: 'skipped'}, + ... + ], + meta: [fields: ['ballotID', 'ward', 'rank1', 'rank2']] + errors:[{ + code: "TooFewFields" + message: "Too few fields: expected 4 fields but parsed 1" + row: 576 + type: "FieldMismatch" + }] + } +*/ + +//const parseCSV = (text : string) => { +// console.log(CSV.parse(text)); +//} // ported from https://github.com/fairvotereform/rcv_cruncher/blob/9bb9f8482290033ff7b31d6b091186474e7afff6/src/rcv_cruncher/parsers.py -export const rankColumnCSV = (text: string) : NewBallotWithVoterID[] => { - parseCSV(text); - return []; +export const rankColumnCSV = ({data, meta, errors}, election) : {output: NewBallotWithVoterID[], errors:object[]} => { + const fields = meta.fields; + + let output = data.map((row,i) => { + +export interface Ballot { + election_id: Uid; //ID of election ballot is cast in + status: string; //Status of string (saved, submitted) + date_submitted: number; //time ballot was submitted, represented as unix timestamp (Date.now()) + votes: Vote[]; // One per poll +} +export interface Vote { + race_id: Uid; // Must match the pollId of the election + scores: Score[]; // One per candidate +} + return { + voter_id: i, + ballot: { + election_id: election_id, + status: 'submitted', + date_submitted: Date.now(), + votes: [] + } + } + }) + + return { + output: [], + errors: [] + }; } \ No newline at end of file From 636ae04a372bd38f8bb15e287246fad7a7550bb6 Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Sun, 12 Jan 2025 17:02:32 -0800 Subject: [PATCH 03/11] Implement parsing and bulk uploading ballots --- .../Controllers/Ballot/castVoteController.ts | 4 +- .../src/Controllers/Roll/voterRollUtils.ts | 15 +-- packages/backend/src/Routes/ballot.routes.ts | 11 +- packages/backend/src/Routes/registerEvents.ts | 14 +-- .../src/components/UploadElections.tsx | 113 ++++++++++++------ .../frontend/src/components/cvrParsers.tsx | 50 ++++---- packages/frontend/src/hooks/useAPI.ts | 6 +- packages/shared/src/domain_model/Ballot.ts | 9 +- 8 files changed, 132 insertions(+), 90 deletions(-) diff --git a/packages/backend/src/Controllers/Ballot/castVoteController.ts b/packages/backend/src/Controllers/Ballot/castVoteController.ts index ddaf9747..3afb50dd 100644 --- a/packages/backend/src/Controllers/Ballot/castVoteController.ts +++ b/packages/backend/src/Controllers/Ballot/castVoteController.ts @@ -46,8 +46,10 @@ async function makeBallotEvent(req: IElectionRequest, targetElection: Election, throw new Unauthorized(missingAuthData); } - roll = await getOrCreateElectionRoll(req, targetElection, req); + // skipping state check since this is allowed when uploading ballots, and it's already explicitly checked for individual ballots + roll = await getOrCreateElectionRoll(req, targetElection, req, voter_id, true); const voterAuthorization = getVoterAuthorization(roll,missingAuthData) + assertVoterMayVote(voterAuthorization, req); //TODO: currently we have both a value on the input Ballot, and the route param. diff --git a/packages/backend/src/Controllers/Roll/voterRollUtils.ts b/packages/backend/src/Controllers/Roll/voterRollUtils.ts index 95dee598..596360d6 100644 --- a/packages/backend/src/Controllers/Roll/voterRollUtils.ts +++ b/packages/backend/src/Controllers/Roll/voterRollUtils.ts @@ -10,7 +10,7 @@ import { hashString } from "../controllerUtils"; const ElectionRollModel = ServiceLocator.electionRollDb(); -export async function getOrCreateElectionRoll(req: IRequest, election: Election, ctx: ILoggingContext): Promise { +export async function getOrCreateElectionRoll(req: IRequest, election: Election, ctx: ILoggingContext, voter_id_override?: string, skipStateCheck?: boolean): Promise { // Checks for existing election roll for user Logger.info(req, `getOrCreateElectionRoll`) const ip_hash = hashString(req.ip!) @@ -23,9 +23,9 @@ export async function getOrCreateElectionRoll(req: IRequest, election: Election, if (election.settings.voter_authentication.voter_id && election.settings.voter_access == 'closed') { // cookies don't support special charaters // https://help.vtex.com/en/tutorial/why-dont-cookies-support-special-characters--6hs7MQzTri6Yg2kQoSICoQ - voter_id = atob(req.cookies?.voter_id); + voter_id = voter_id_override ?? atob(req.cookies?.voter_id); } else if (election.settings.voter_authentication.voter_id && election.settings.voter_access == 'open') { - voter_id = req.user?.sub + voter_id = voter_id_override ?? req.user?.sub } // Get all election roll entries that match any of the voter authentication fields @@ -38,12 +38,9 @@ export async function getOrCreateElectionRoll(req: IRequest, election: Election, if (electionRollEntries == null) { // No election roll found, create one if voter access is open and election state is open - if (election.settings.voter_access !== 'open') { - return null - } - if (election.state !== 'open') { - return null - } + if (election.settings.voter_access !== 'open') return null + if (!skipStateCheck && election.state !== 'open') return null + Logger.info(req, "Creating new roll"); const new_voter_id = election.settings.voter_authentication.voter_id ? voter_id : randomUUID() const history = [{ diff --git a/packages/backend/src/Routes/ballot.routes.ts b/packages/backend/src/Routes/ballot.routes.ts index c2665418..8ad54edf 100644 --- a/packages/backend/src/Routes/ballot.routes.ts +++ b/packages/backend/src/Routes/ballot.routes.ts @@ -245,16 +245,7 @@ ballotRouter.post('/Election/:id/vote', asyncHandler(castVoteController)) * type: array * items: * type: object - * properties: - * voter_id: - * type: string - * description: id of voter - * success: - * type: boolean - * description: If ballot was uploaded - * message: - * type: string - * description: Corresponding message + * $ref: '#/components/schemas/BallotSubmitStatus' * 404: * description: Election not found */ ballotRouter.post('/Election/:id/uploadBallots', asyncHandler(uploadBallotsController)) diff --git a/packages/backend/src/Routes/registerEvents.ts b/packages/backend/src/Routes/registerEvents.ts index 73e898c7..7160184e 100644 --- a/packages/backend/src/Routes/registerEvents.ts +++ b/packages/backend/src/Routes/registerEvents.ts @@ -7,11 +7,11 @@ const { handleSendInviteEvent } = require('../Controllers/Election/sendInvitesCo const { handleSendEmailEvent } =require('../Controllers/Election/sendEmailController'); export default async function registerEvents() { - const ctx = Logger.createContext("app init"); - Logger.debug(ctx, "registering events"); - const eventQueue = await ServiceLocator.eventQueue(); - eventQueue.subscribe("castVoteEvent", handleCastVoteEvent); - eventQueue.subscribe("sendInviteEvent", handleSendInviteEvent); - eventQueue.subscribe("sendEmailEvent", handleSendEmailEvent); - Logger.debug(ctx, "registering events complete"); + //const ctx = Logger.createContext("app init"); + //Logger.debug(ctx, "registering events"); + //const eventQueue = await ServiceLocator.eventQueue(); + //eventQueue.subscribe("castVoteEvent", handleCastVoteEvent); + //eventQueue.subscribe("sendInviteEvent", handleSendInviteEvent); + //eventQueue.subscribe("sendEmailEvent", handleSendEmailEvent); + //Logger.debug(ctx, "registering events complete"); } \ No newline at end of file diff --git a/packages/frontend/src/components/UploadElections.tsx b/packages/frontend/src/components/UploadElections.tsx index 13f7a4c9..bc8bd076 100644 --- a/packages/frontend/src/components/UploadElections.tsx +++ b/packages/frontend/src/components/UploadElections.tsx @@ -6,7 +6,6 @@ import EnhancedTable from "./EnhancedTable"; import { rankColumnCSV } from "./cvrParsers"; import { v4 as uuidv4 } from 'uuid'; import Papa from 'papaparse'; -import { usePostElection } from "~/hooks/useAPI"; import useAuthSession from "./AuthSessionContextProvider"; import { defaultElection } from "./ElectionForm/CreateElectionDialog"; import { Candidate } from "@equal-vote/star-vote-shared/domain_model/Candidate"; @@ -17,10 +16,8 @@ export default () => { const {t} = useSubstitutedTranslation(); const inputRef = useRef(null) const [electionsSubmitted, setElectionsSubmitted] = useState(false); - const { error: postError, isPending, makeRequest: postElection } = usePostElection() const authSession = useAuthSession() - const submitElections = () => { setElectionsSubmitted(true) @@ -29,6 +26,7 @@ export default () => { const post_process = async (parsed_csv) => { // #2 : Infer Election Settings const errorRows = new Set(parsed_csv.errors.map(error => error.row)) + // NOTE: this assumes rank_column_csv, may not work with other formats const rankFields = parsed_csv.meta.fields.filter((field:string) => field.startsWith('rank')); const maxRankings = rankFields.length; let candidateNames = new Set(); @@ -41,45 +39,91 @@ export default () => { // TODO: infer num winners // #3 : Create (or fetch) Election - const election = await postElection({ - Election: { - ...defaultElection, - title: cvr.name.split('.')[0], - state: 'closed', - owner_id: authSession.getIdField('sub'), - settings: { - ...defaultElection.settings, - max_rankings: maxRankings - }, - races: [ - { - race_id: uuidv4(), - voting_method: 'IRV', - title: cvr.name.split('.')[0], - candidates: [...candidateNames].map(name => ({ - candidate_id: uuidv4(), - candidate_name: name - })) as Candidate[], - num_winners: 1 - } - ] + // NOTE: I'm not using usePostElection because I need to handle multiple requests + const postElectionRes = await fetch('/API/Elections', { + method: 'post', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json', }, + body: JSON.stringify({ + Election: { + ...defaultElection, + title: cvr.name.split('.')[0], + state: 'closed', + owner_id: authSession.getIdField('sub'), + settings: { + ...defaultElection.settings, + max_rankings: maxRankings, + voter_access: 'open' + }, + races: [ + { + race_id: uuidv4(), + voting_method: 'IRV', + title: cvr.name.split('.')[0], + candidates: [...candidateNames].map(name => ({ + candidate_id: uuidv4(), + candidate_name: name + })) as Candidate[], + num_winners: 1 + } + ] + }, + }) }) - if (!election){ + if (!postElectionRes.ok){ parsed_csv.errors.push({ code: "ElectionCreationFailed", - message: postError, + message: `Error making request: ${postElectionRes.status.toString()}`, row: -1, type: "ElectionCreationFailed" }) return; }; - //// #4 : Convert Rows to Ballots - //rankColumnCSV(results, 'election_id') - // TODO: store results.errors somewhere - + const {election} = await postElectionRes.json() + + // #4 : Convert Rows to Ballots + let {ballots, errors} = rankColumnCSV(parsed_csv, election) + + // #5 : Upload Ballots + const batchSize = 100; + let batchIndex = -1; + let responses = []; + // TODO: this batching isn't ideal since it'll be tricky to recovered from a partial failure + // that said this will mainly be relevant when uploading batches for an existing election so I'll leave it for now + let filteredBallots = ballots.filter((b, i) => !errorRows.has(i)); + while((batchIndex+1)*batchSize < ballots.length && batchIndex < 1000 /* a dummy check to avoid infinite loops*/){ + batchIndex++; + const uploadRes = await fetch(`/API/Election/${election.election_id}/uploadBallots`, { + method: 'post', + headers: { + 'Accept': 'application/json', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ballots: filteredBallots.slice(batchIndex*batchSize, (batchIndex+1)*batchSize)}) + }) + + if (!uploadRes.ok){ + errors.push({ + code: "UploadBallotsFailed", + message: `Error making request: ${uploadRes.status.toString()}`, + row: -1, + type: "UploadBallotsFailed" + }) + console.log(errors); + return; + }; + + let res = await uploadRes.json(); + responses = [...responses, ...res.responses]; + } + console.log(responses); + + // TODO: display error list somewhere + console.log('SUCCESS!: ', election.election_id); } Papa.parse(cvr.url, { header: true, @@ -87,12 +131,7 @@ export default () => { dynamicTyping: true, complete: post_process }) - // infer election settings from name, and csv headers - // create election (w/ first race) - // convert rows to ballots - // upload ballots - - // optional: add more election settings based on ballots (ex. max rankings) + }) } diff --git a/packages/frontend/src/components/cvrParsers.tsx b/packages/frontend/src/components/cvrParsers.tsx index e2250cef..088248cf 100644 --- a/packages/frontend/src/components/cvrParsers.tsx +++ b/packages/frontend/src/components/cvrParsers.tsx @@ -1,4 +1,5 @@ import { NewBallotWithVoterID } from "@equal-vote/star-vote-shared/domain_model/Ballot"; +import { Election } from "@equal-vote/star-vote-shared/domain_model/Election"; /* Example Input (assuming papa parse) @@ -19,39 +20,40 @@ import { NewBallotWithVoterID } from "@equal-vote/star-vote-shared/domain_model/ } */ -//const parseCSV = (text : string) => { -// console.log(CSV.parse(text)); -//} - // ported from https://github.com/fairvotereform/rcv_cruncher/blob/9bb9f8482290033ff7b31d6b091186474e7afff6/src/rcv_cruncher/parsers.py -export const rankColumnCSV = ({data, meta, errors}, election) : {output: NewBallotWithVoterID[], errors:object[]} => { - const fields = meta.fields; - - let output = data.map((row,i) => { +export const rankColumnCSV = ({data, meta, errors}, election: Election) : {ballots: NewBallotWithVoterID[], errors:object[]} => { + const errorRows = new Set(errors.map(error => error.row)) + const rankFields = meta.fields.filter((field:string) => field.startsWith('rank')); -export interface Ballot { - election_id: Uid; //ID of election ballot is cast in - status: string; //Status of string (saved, submitted) - date_submitted: number; //time ballot was submitted, represented as unix timestamp (Date.now()) - votes: Vote[]; // One per poll -} -export interface Vote { - race_id: Uid; // Must match the pollId of the election - scores: Score[]; // One per candidate -} + let ballots = data.map((row,i) => { + if(errorRows.has(i)) return; + // TODO: this currently doesn't handle overvotes or duplicate ranks + // TODO: add try catch for adding errors + let invRow = rankFields.reduce((obj, key) => { + obj[row[key]] = Number(key.replace('rank', '')); + return obj; + }, {}) return { voter_id: i, ballot: { - election_id: election_id, + election_id: election.election_id, status: 'submitted', date_submitted: Date.now(), - votes: [] + votes: [ + { + race_id: election.races[0].race_id, + scores: election.races[0].candidates.map(c => { + let ranking = invRow[c.candidate_name]; + return { + candidate_id: c.candidate_id, + score: ranking ? ranking : null + } + }) + } + ] } } }) - return { - output: [], - errors: [] - }; + return {ballots, errors}; } \ No newline at end of file diff --git a/packages/frontend/src/hooks/useAPI.ts b/packages/frontend/src/hooks/useAPI.ts index b3234f6b..d45a5285 100644 --- a/packages/frontend/src/hooks/useAPI.ts +++ b/packages/frontend/src/hooks/useAPI.ts @@ -4,7 +4,7 @@ import { ElectionRoll } from "@equal-vote/star-vote-shared/domain_model/Election import useFetch from "./useFetch"; import { VotingMethod } from "@equal-vote/star-vote-shared/domain_model/Race"; import { ElectionResults } from "@equal-vote/star-vote-shared/domain_model/ITabulators"; -import { Ballot, NewBallot, AnonymizedBallot } from "@equal-vote/star-vote-shared/domain_model/Ballot"; +import { Ballot, NewBallot, AnonymizedBallot, NewBallotWithVoterID, BallotSubmitStatus } from "@equal-vote/star-vote-shared/domain_model/Ballot"; import { email_request_data } from "@equal-vote/star-vote-backend/src/Controllers/Election/sendEmailController" export const useGetElection = (electionID: string | undefined) => { @@ -131,6 +131,10 @@ export const usePostBallot = (election_id: string | undefined) => { return useFetch<{ ballot: NewBallot, receiptEmail?: string }, {ballot: Ballot}>(`/API/Election/${election_id}/vote`, 'post') } +export const useUploadBallots = (election_id: string | undefined) => { + return useFetch<{ ballots: NewBallotWithVoterID[] }, {responses: BallotSubmitStatus[]}>(`/API/Election/${election_id}/uploadBallots`, 'post') +} + export const useGetSandboxResults = () => { return useFetch<{ cvr: number[][], diff --git a/packages/shared/src/domain_model/Ballot.ts b/packages/shared/src/domain_model/Ballot.ts index f2b2539e..76afa4ee 100644 --- a/packages/shared/src/domain_model/Ballot.ts +++ b/packages/shared/src/domain_model/Ballot.ts @@ -36,6 +36,12 @@ export interface BallotAction { timestamp:number; } +export interface BallotSubmitStatus { + voter_id:string; + success:boolean; + message:string; +} + export interface NewBallot extends PartialBy {} export function ballotValidation(election: Election, obj:Ballot): string | null { @@ -76,7 +82,8 @@ export function ballotValidation(election: Election, obj:Ballot): string | null if (['RankedRobin', 'IRV', 'STV'].includes(race.voting_method)) { const numCandidates = race.candidates.length; vote.scores.forEach(score => { - if (score && score.score > numCandidates || (maxRankings && score.score > maxRankings) || score.score < 0) { + // Arend: Removing check against numCandidates, that's not necessarily true for public RCV elections + if (score && /*score.score > numCandidates ||*/ (maxRankings && score.score > maxRankings) || score.score < 0) { outOfBoundsError += `Race: ${race.title}, Score: ${score.score}; `; } }) From 1351ca7d766d8093d5da66fc42d68de9a59d0e1b Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Sun, 12 Jan 2025 17:04:51 -0800 Subject: [PATCH 04/11] Comments --- packages/backend/src/Routes/registerEvents.ts | 14 +++++++------- .../Results/components/VoterIntentWidget.tsx | 1 - 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/Routes/registerEvents.ts b/packages/backend/src/Routes/registerEvents.ts index 7160184e..73e898c7 100644 --- a/packages/backend/src/Routes/registerEvents.ts +++ b/packages/backend/src/Routes/registerEvents.ts @@ -7,11 +7,11 @@ const { handleSendInviteEvent } = require('../Controllers/Election/sendInvitesCo const { handleSendEmailEvent } =require('../Controllers/Election/sendEmailController'); export default async function registerEvents() { - //const ctx = Logger.createContext("app init"); - //Logger.debug(ctx, "registering events"); - //const eventQueue = await ServiceLocator.eventQueue(); - //eventQueue.subscribe("castVoteEvent", handleCastVoteEvent); - //eventQueue.subscribe("sendInviteEvent", handleSendInviteEvent); - //eventQueue.subscribe("sendEmailEvent", handleSendEmailEvent); - //Logger.debug(ctx, "registering events complete"); + const ctx = Logger.createContext("app init"); + Logger.debug(ctx, "registering events"); + const eventQueue = await ServiceLocator.eventQueue(); + eventQueue.subscribe("castVoteEvent", handleCastVoteEvent); + eventQueue.subscribe("sendInviteEvent", handleSendInviteEvent); + eventQueue.subscribe("sendEmailEvent", handleSendEmailEvent); + Logger.debug(ctx, "registering events complete"); } \ No newline at end of file diff --git a/packages/frontend/src/components/Election/Results/components/VoterIntentWidget.tsx b/packages/frontend/src/components/Election/Results/components/VoterIntentWidget.tsx index 6adac543..d8b71815 100644 --- a/packages/frontend/src/components/Election/Results/components/VoterIntentWidget.tsx +++ b/packages/frontend/src/components/Election/Results/components/VoterIntentWidget.tsx @@ -96,7 +96,6 @@ export default ({eliminationOrderById, winnerId} : {eliminationOrderById : strin if(trailingRanks) return 4; return 2; } - //if(ballotType() == 2) console.log(loggedBallot); data[ballotType()-1].votes++; }) From f5124ba5a978a48cc428ccbd18e4949335b795cc Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Sun, 12 Jan 2025 22:53:01 -0800 Subject: [PATCH 05/11] Change color of closed chip --- .../src/components/ElectionForm/Details/ElectionStateChip.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/frontend/src/components/ElectionForm/Details/ElectionStateChip.tsx b/packages/frontend/src/components/ElectionForm/Details/ElectionStateChip.tsx index 9781fcf4..daea06d8 100644 --- a/packages/frontend/src/components/ElectionForm/Details/ElectionStateChip.tsx +++ b/packages/frontend/src/components/ElectionForm/Details/ElectionStateChip.tsx @@ -14,7 +14,8 @@ const getStateColor = (state: string) => { case 'open': return 'blue'; case 'closed': - return 'red'; + // changed from red, since being closed isn't a bad thing. It just means the voting period is finished + return 'orange'; case 'archived': return 'gray4'; // Voted From 6bb8808f0979cac9ab0d7876e0dfb8f779421a2b Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Tue, 14 Jan 2025 15:49:44 -0800 Subject: [PATCH 06/11] Update swagger --- packages/backend/src/OpenApi/swagger.json | 35 +++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/packages/backend/src/OpenApi/swagger.json b/packages/backend/src/OpenApi/swagger.json index c0eec06f..71edf8d6 100644 --- a/packages/backend/src/OpenApi/swagger.json +++ b/packages/backend/src/OpenApi/swagger.json @@ -137,6 +137,25 @@ ], "type": "object" }, + "BallotSubmitStatus": { + "properties": { + "message": { + "type": "string" + }, + "success": { + "type": "boolean" + }, + "voter_id": { + "type": "string" + } + }, + "required": [ + "message", + "success", + "voter_id" + ], + "type": "object" + }, "Candidate": { "properties": { "bio": { @@ -729,6 +748,21 @@ ], "type": "object" }, + "NewBallotWithVoterID": { + "properties": { + "ballot": { + "$ref": "#/components/schemas/NewBallot" + }, + "voter_id": { + "type": "string" + } + }, + "required": [ + "ballot", + "voter_id" + ], + "type": "object" + }, "NewElection": { "properties": { "admin_ids": { @@ -1152,6 +1186,7 @@ "canSendEmails", "canUnflagElectionRoll", "canUpdatePublicArchive", + "canUploadBallots", "canViewBallot", "canViewBallots", "canViewElection", From ccb68c20efd9d2fa662bd33a9606463677895fbd Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Wed, 29 Jan 2025 14:39:39 -0800 Subject: [PATCH 07/11] Add ballot_source and public_archive_id --- packages/backend/sample.env | 4 +-- .../src/Migrations/2025_01_29_admin_upload.ts | 31 +++++++++++++++++++ packages/backend/src/Models/Elections.ts | 1 - .../ElectionForm/CreateElectionDialog.tsx | 12 +++---- .../src/components/ElectionForm/QuickPoll.tsx | 13 +++----- .../src/components/UploadElections.tsx | 5 ++- packages/shared/src/domain_model/Election.ts | 12 ++----- 7 files changed, 48 insertions(+), 30 deletions(-) create mode 100644 packages/backend/src/Migrations/2025_01_29_admin_upload.ts diff --git a/packages/backend/sample.env b/packages/backend/sample.env index 215dc153..5497092f 100644 --- a/packages/backend/sample.env +++ b/packages/backend/sample.env @@ -14,8 +14,8 @@ ALLOWED_URLS='http://localhost:3000' # 3000 should match FRONTEND_PORT from fron BACKEND_PORT=5000 # if updated, make sure to also change the proxy and socket urls in the frontend .env #### FRONT PAGE STATS #### -CLASSIC_ELECTION_COUNT=500 -CLASSIC_VOTE_COUNT=5000 +CLASSIC_ELECTION_COUNT=0 +CLASSIC_VOTE_COUNT=0 #### EMAIL #### # Contact elections@equal.vote if you need access for developing email features diff --git a/packages/backend/src/Migrations/2025_01_29_admin_upload.ts b/packages/backend/src/Migrations/2025_01_29_admin_upload.ts new file mode 100644 index 00000000..8b277d99 --- /dev/null +++ b/packages/backend/src/Migrations/2025_01_29_admin_upload.ts @@ -0,0 +1,31 @@ +import { Kysely } from 'kysely' + +export async function up(db: Kysely): Promise { + await db.schema.alterTable('electionDB') + /* ballot_source types + live_election: Ballots submitted by voters during election + prior_election: Election admin uploaded ballots from a previous election + */ + .addColumn('ballot_source', 'varchar' ) + // unique identifier for mapping public archive elections to their real elections + // ex. Genola_11022021_CityCouncil + .addColumn('public_archive_id', 'varchar' ) + // support_email is obsolete, it has been superceded by settings.contact_email + .dropColumn('support_email') + .execute() + + await db.updateTable('electionDB') + .set({ + ballot_source: 'live_election', + public_archive_id: '', + }) + .execute() +} + +export async function down(db: Kysely): Promise { + await db.schema.alterTable('electionDB') + .dropColumn('ballot_source') + .dropColumn('public_archive_id') + .addColumn('support_email', 'varchar') + .execute() +} \ No newline at end of file diff --git a/packages/backend/src/Models/Elections.ts b/packages/backend/src/Models/Elections.ts index bdea5f0d..def0c88f 100644 --- a/packages/backend/src/Models/Elections.ts +++ b/packages/backend/src/Models/Elections.ts @@ -113,7 +113,6 @@ export default class ElectionsDB implements IElectionStore { ) } - const elections = query.execute().catch(dneCatcher) return elections diff --git a/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx b/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx index 14f8874b..2c708903 100644 --- a/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx +++ b/packages/frontend/src/components/ElectionForm/CreateElectionDialog.tsx @@ -1,19 +1,14 @@ -import { Box, Button, capitalize, Dialog, DialogActions, DialogContent, DialogTitle, Divider, FormControlLabel, IconButton, MenuItem, Radio, RadioGroup, Select, Step, StepConnector, StepContent, StepLabel, Stepper, TextField, Tooltip, Typography } from "@mui/material"; +import { Box, capitalize, Dialog, DialogActions, DialogContent, DialogTitle, FormControlLabel, Radio, RadioGroup, Step, StepContent, StepLabel, Stepper, TextField, Typography } from "@mui/material"; import { StyledButton, Tip } from "../styles"; -import { Dispatch, SetStateAction, createContext, useContext, useEffect, useRef, useState } from "react"; +import { createContext, useContext, useEffect, useState } from "react"; import { ElectionTitleField } from "./Details/ElectionDetailsForm"; -import ArrowForwardIosIcon from '@mui/icons-material/ArrowForwardIos'; -import ExpandLess from '@mui/icons-material/ExpandLess' -import ExpandMore from '@mui/icons-material/ExpandMore' -import { openFeedback, RowButtonWithArrow, useSubstitutedTranslation } from "../util"; -import { useLocalStorage } from "~/hooks/useLocalStorage"; +import { RowButtonWithArrow, useSubstitutedTranslation } from "../util"; import { NewElection } from "@equal-vote/star-vote-shared/domain_model/Election"; import { DateTime } from "luxon"; import useAuthSession from "../AuthSessionContextProvider"; import { usePostElection } from "~/hooks/useAPI"; import { TermType } from "@equal-vote/star-vote-shared/domain_model/ElectionSettings"; import { useNavigate } from "react-router"; -import { useTranslation } from "react-i18next"; import { TimeZone } from "@equal-vote/star-vote-shared/domain_model/Util"; /////// PROVIDER SETUP ///// @@ -55,6 +50,7 @@ export const defaultElection: NewElection = { description: '', state: 'draft', frontend_url: '', + ballot_source: 'live_election', races: [], settings: { voter_authentication: { diff --git a/packages/frontend/src/components/ElectionForm/QuickPoll.tsx b/packages/frontend/src/components/ElectionForm/QuickPoll.tsx index 8e7c6cfc..b01ea32a 100644 --- a/packages/frontend/src/components/ElectionForm/QuickPoll.tsx +++ b/packages/frontend/src/components/ElectionForm/QuickPoll.tsx @@ -1,15 +1,12 @@ -import React, { useContext, useState } from 'react' -import Container from '@mui/material/Container'; -import Grid from "@mui/material/Grid"; -import TextField from "@mui/material/TextField"; -import { useNavigate } from "react-router" +import { useContext, useState } from 'react'; +import { useNavigate } from "react-router"; import structuredClone from '@ungap/structured-clone'; -import { StyledButton, StyledTextField } from '../styles.js' +import { StyledButton, StyledTextField } from '../styles.js'; import { Box, Button, IconButton, MenuItem, Select, SelectChangeEvent, Typography } from '@mui/material'; import DeleteIcon from '@mui/icons-material/Delete'; import { usePostElection } from '../../hooks/useAPI'; import { useCookie } from '../../hooks/useCookie'; -import { Election, NewElection } from '@equal-vote/star-vote-shared/domain_model/Election'; +import { NewElection } from '@equal-vote/star-vote-shared/domain_model/Election'; import { CreateElectionContext } from './CreateElectionDialog.js'; import useSnackbar from '../SnackbarContext.js'; @@ -39,6 +36,7 @@ const QuickPoll = () => { frontend_url: '', owner_id: '0', is_public: false, + ballot_source: 'live_election', races: [ { title: '', @@ -75,7 +73,6 @@ const QuickPoll = () => { } } - const [election, setElectionData] = useState(QuickPollTemplate) const onSubmitElection = async (election) => { // calls post election api, throws error if response not ok diff --git a/packages/frontend/src/components/UploadElections.tsx b/packages/frontend/src/components/UploadElections.tsx index bc8bd076..261be861 100644 --- a/packages/frontend/src/components/UploadElections.tsx +++ b/packages/frontend/src/components/UploadElections.tsx @@ -9,6 +9,7 @@ import Papa from 'papaparse'; import useAuthSession from "./AuthSessionContextProvider"; import { defaultElection } from "./ElectionForm/CreateElectionDialog"; import { Candidate } from "@equal-vote/star-vote-shared/domain_model/Candidate"; +import { NewElection } from '@equal-vote/star-vote-shared/domain_model/Election'; export default () => { const [addToPublicArchive, setAddToPublicArchive] = useState(false) @@ -52,6 +53,8 @@ export default () => { title: cvr.name.split('.')[0], state: 'closed', owner_id: authSession.getIdField('sub'), + ballot_source: 'prior_election', + public_archive_id: addToPublicArchive? cvr.name.split('.')[0] : undefined, settings: { ...defaultElection.settings, max_rankings: maxRankings, @@ -69,7 +72,7 @@ export default () => { num_winners: 1 } ] - }, + } as NewElection, }) }) diff --git a/packages/shared/src/domain_model/Election.ts b/packages/shared/src/domain_model/Election.ts index 26ba9d50..e78348ce 100644 --- a/packages/shared/src/domain_model/Election.ts +++ b/packages/shared/src/domain_model/Election.ts @@ -14,7 +14,6 @@ export interface Election { frontend_url: string; // base URL for the frontend start_time?: Date | string; // when the election starts end_time?: Date | string; // when the election ends - support_email?: string; // email available to voters to request support owner_id: Uid; // user_id of owner of election audit_ids?: Uid[]; // user_id of account with audit access admin_ids?: Uid[]; // user_id of account with admin access @@ -28,16 +27,14 @@ export interface Election { create_date: Date | string; // Date this object was created update_date: Date | string; // Date this object was last updated head: boolean;// Head version of this object + ballot_source: 'live_election' | 'prior_election'; + public_archive_id?: string; } type Omit = Pick> export type PartialBy = Omit & Partial> export interface NewElection extends PartialBy {} - - - - export function electionValidation(obj:Election): string | null { if (!obj){ return "Election is null"; @@ -71,11 +68,6 @@ export function electionValidation(obj:Election): string | null { return "Invalid End Time Date Format"; } } - if (obj.support_email) { - if (!emailRegex.test(obj.support_email)) { - return "Invalid Support Email Format"; - } - } if (typeof obj.owner_id !== 'string'){ return "Invalid Owner ID"; } From 0c71490211b6fd5d75b8fdc3d1a284ec19b61cc8 Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Wed, 29 Jan 2025 15:03:49 -0800 Subject: [PATCH 08/11] Add public archive query --- .../Election/getElectionsController.ts | 7 ++- .../src/Migrations/2025_01_29_admin_upload.ts | 2 +- packages/backend/src/Models/Elections.ts | 29 ++++++++-- packages/backend/src/OpenApi/swagger.json | 53 +++++++++---------- packages/backend/src/test/database_sandbox.ts | 3 +- .../components/Election/Admin/ViewBallots.tsx | 1 + .../src/components/ElectionForm/QuickPoll.tsx | 2 +- 7 files changed, 58 insertions(+), 39 deletions(-) diff --git a/packages/backend/src/Controllers/Election/getElectionsController.ts b/packages/backend/src/Controllers/Election/getElectionsController.ts index ada081b6..75a6cb6b 100644 --- a/packages/backend/src/Controllers/Election/getElectionsController.ts +++ b/packages/backend/src/Controllers/Election/getElectionsController.ts @@ -9,6 +9,7 @@ import { Election, removeHiddenFields } from '@equal-vote/star-vote-shared/domai var ElectionsModel = ServiceLocator.electionsDb(); var ElectionRollModel = ServiceLocator.electionRollDb(); +// TODO: We should probably split this up as the user will only need one of these filters const getElections = async (req: IElectionRequest, res: Response, next: NextFunction) => { Logger.info(req, `getElections`); // var filter = (req.query.filter == undefined) ? "" : req.query.filter; @@ -51,14 +52,12 @@ const getElections = async (req: IElectionRequest, res: Response, next: NextFunc } } - /////////// OPEN ELECTIONS //////////////// - var open_elections = await ElectionsModel.getOpenElections(req); - res.json({ elections_as_official, elections_as_unsubmitted_voter, elections_as_submitted_voter, - open_elections + public_archive_elections: await ElectionsModel.getPublicArchiveElections(req), + open_elections: await ElectionsModel.getOpenElections(req) }); } diff --git a/packages/backend/src/Migrations/2025_01_29_admin_upload.ts b/packages/backend/src/Migrations/2025_01_29_admin_upload.ts index 8b277d99..ca9c0fa7 100644 --- a/packages/backend/src/Migrations/2025_01_29_admin_upload.ts +++ b/packages/backend/src/Migrations/2025_01_29_admin_upload.ts @@ -17,7 +17,7 @@ export async function up(db: Kysely): Promise { await db.updateTable('electionDB') .set({ ballot_source: 'live_election', - public_archive_id: '', + public_archive_id: null, }) .execute() } diff --git a/packages/backend/src/Models/Elections.ts b/packages/backend/src/Models/Elections.ts index def0c88f..0de63a41 100644 --- a/packages/backend/src/Models/Elections.ts +++ b/packages/backend/src/Models/Elections.ts @@ -95,6 +95,19 @@ export default class ElectionsDB implements IElectionStore { }); } + async getPublicArchiveElections(ctx: ILoggingContext): Promise { + Logger.debug(ctx, `${tableName}.getPublicArchiveElections`); + // Returns all elections where settings.voter_access == open and state == open + + // TODO: The filter is pretty inefficient for now since I don't think there's a way to include on settings.voter_access in the query + return await this._postgresClient + .selectFrom(tableName) + .where('head', '=', true) + .where('public_archive_id', '!=', null) + .selectAll() + .execute() + } + getElections(id: string, email: string, ctx: ILoggingContext): Promise { // When I filter in trello it adds "filter=member:arendpetercastelein,overdue:true" to the URL, I'm following the same pattern here Logger.debug(ctx, `${tableName}.getAll ${id}`); @@ -113,12 +126,22 @@ export default class ElectionsDB implements IElectionStore { ) } - const elections = query.execute().catch(dneCatcher) + return query.execute().catch(dneCatcher) + } - return elections + getElectionsSourcedFromPrior(ctx: ILoggingContext): Promise { + // When I filter in trello it adds "filter=member:arendpetercastelein,overdue:true" to the URL, I'm following the same pattern here + Logger.debug(ctx, `${tableName}.getSourcedFromPrior`); + + return this._postgresClient + .selectFrom(tableName) + .where('ballot_source', '=', 'prior_election') + .where('head', '=', true) + .selectAll() + .execute() + .catch(dneCatcher); } - // TODO: I'm a bit lazy for now just having Object as the type getBallotCountsForAllElections(ctx: ILoggingContext): Promise { Logger.debug(ctx, `${tableName}.getAllElectionsWithBallotCounts`); diff --git a/packages/backend/src/OpenApi/swagger.json b/packages/backend/src/OpenApi/swagger.json index 71edf8d6..99cacd0c 100644 --- a/packages/backend/src/OpenApi/swagger.json +++ b/packages/backend/src/OpenApi/swagger.json @@ -224,6 +224,13 @@ "auth_key": { "type": "string" }, + "ballot_source": { + "enum": [ + "live_election", + "prior_election" + ], + "type": "string" + }, "claim_key_hash": { "type": "string" }, @@ -273,6 +280,9 @@ "owner_id": { "type": "string" }, + "public_archive_id": { + "type": "string" + }, "races": { "items": { "$ref": "#/components/schemas/Race" @@ -296,9 +306,6 @@ "state": { "$ref": "#/components/schemas/ElectionState" }, - "support_email": { - "type": "string" - }, "title": { "type": "string" }, @@ -315,6 +322,7 @@ } }, "required": [ + "ballot_source", "create_date", "election_id", "frontend_url", @@ -780,6 +788,13 @@ "auth_key": { "type": "string" }, + "ballot_source": { + "enum": [ + "live_election", + "prior_election" + ], + "type": "string" + }, "claim_key_hash": { "type": "string" }, @@ -829,6 +844,9 @@ "owner_id": { "type": "string" }, + "public_archive_id": { + "type": "string" + }, "races": { "items": { "$ref": "#/components/schemas/Race" @@ -852,9 +870,6 @@ "state": { "$ref": "#/components/schemas/ElectionState" }, - "support_email": { - "type": "string" - }, "title": { "type": "string" }, @@ -871,6 +886,7 @@ } }, "required": [ + "ballot_source", "frontend_url", "owner_id", "races", @@ -2640,15 +2656,7 @@ "type": "array", "items": { "type": "object", - "properties": { - "ballot": { - "type": "object", - "$ref": "#/components/schemas/NewBallot" - }, - "voter_id": { - "type": "string" - } - } + "$ref": "#/components/schemas/NewBallotWithVoterID" } } } @@ -2668,20 +2676,7 @@ "type": "array", "items": { "type": "object", - "properties": { - "voter_id": { - "type": "string", - "description": "id of voter" - }, - "success": { - "type": "boolean", - "description": "If ballot was uploaded" - }, - "message": { - "type": "string", - "description": "Corresponding message" - } - } + "$ref": "#/components/schemas/BallotSubmitStatus" } } } diff --git a/packages/backend/src/test/database_sandbox.ts b/packages/backend/src/test/database_sandbox.ts index c6084591..2945f225 100644 --- a/packages/backend/src/test/database_sandbox.ts +++ b/packages/backend/src/test/database_sandbox.ts @@ -23,7 +23,8 @@ function buildElection(i: string, update_date: string, head: boolean): Election } }, update_date: update_date, - head: head + head: head, + ballot_source: 'live_election', } } diff --git a/packages/frontend/src/components/Election/Admin/ViewBallots.tsx b/packages/frontend/src/components/Election/Admin/ViewBallots.tsx index 1e46dbd8..b40d64bd 100644 --- a/packages/frontend/src/components/Election/Admin/ViewBallots.tsx +++ b/packages/frontend/src/components/Election/Admin/ViewBallots.tsx @@ -18,6 +18,7 @@ const ViewBallots = () => { // so we use election instead of precinctFilteredElection const { election } = useElection() const { data, isPending, error, makeRequest: fetchBallots } = useGetBallots(election.election_id) + const flags = useFeatureFlags(); useEffect(() => { fetchBallots() }, []) const [isViewing, setIsViewing] = useState(false) diff --git a/packages/frontend/src/components/ElectionForm/QuickPoll.tsx b/packages/frontend/src/components/ElectionForm/QuickPoll.tsx index b01ea32a..9ae4202d 100644 --- a/packages/frontend/src/components/ElectionForm/QuickPoll.tsx +++ b/packages/frontend/src/components/ElectionForm/QuickPoll.tsx @@ -23,7 +23,7 @@ const QuickPoll = () => { const {t} = useSubstitutedTranslation('poll'); - // TODO: we may edit the db entries in the future so that these align + // TODO: we may edit the db entries in the future so that these align const dbKeys = { 'star': 'STAR', 'approval': 'Approval', From a42cff219419ee9dbeda9fbea4ca785970429f8d Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Wed, 29 Jan 2025 15:12:54 -0800 Subject: [PATCH 09/11] Filter sourceFromPrior when calculating ballot counts --- .../Election/getElectionsController.ts | 19 +++++++++++++------ packages/backend/src/Models/Elections.ts | 3 +++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/Controllers/Election/getElectionsController.ts b/packages/backend/src/Controllers/Election/getElectionsController.ts index 75a6cb6b..632b01d1 100644 --- a/packages/backend/src/Controllers/Election/getElectionsController.ts +++ b/packages/backend/src/Controllers/Election/getElectionsController.ts @@ -66,18 +66,25 @@ const innerGetGlobalElectionStats = async (req: IRequest) => { let electionVotes = await ElectionsModel.getBallotCountsForAllElections(req); + let sourcedFromPrior = await ElectionsModel.getElectionsSourcedFromPrior(req); + let priorElections = sourcedFromPrior?.map(e => e.election_id) ?? []; + let stats = { elections: Number(process.env.CLASSIC_ELECTION_COUNT ?? 0), votes: Number(process.env.CLASSIC_VOTE_COUNT ?? 0), }; - electionVotes?.map(m => m['v'])?.forEach((count) => { - stats['votes'] = stats['votes'] + Number(count); - if(count >= 2){ - stats['elections'] = stats['elections'] + 1; + electionVotes + ?.filter(m => !priorElections.includes(m['election_id'])) + ?.map(m => m['v']) + ?.forEach((count) => { + stats['votes'] = stats['votes'] + Number(count); + if(count >= 2){ + stats['elections'] = stats['elections'] + 1; + } + return stats; } - return stats; - }); + ); return stats; } diff --git a/packages/backend/src/Models/Elections.ts b/packages/backend/src/Models/Elections.ts index 0de63a41..80673ede 100644 --- a/packages/backend/src/Models/Elections.ts +++ b/packages/backend/src/Models/Elections.ts @@ -11,6 +11,7 @@ import { InternalServerError } from '@curveball/http-errors'; const tableName = 'electionDB'; interface IVoteCount{ + election_id: string; v: number; } @@ -142,6 +143,7 @@ export default class ElectionsDB implements IElectionStore { .catch(dneCatcher); } + // TODO: this function should probably be in the ballots model getBallotCountsForAllElections(ctx: ILoggingContext): Promise { Logger.debug(ctx, `${tableName}.getAllElectionsWithBallotCounts`); @@ -151,6 +153,7 @@ export default class ElectionsDB implements IElectionStore { .select( (eb) => eb.fn.count('ballot_id').as('v') ) + .select('election_id') .where('head', '=', true) .groupBy('election_id') .orderBy('election_id') From 6dc79b71c779f65c937ba9a32c2f9bf2555601cb Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Wed, 29 Jan 2025 15:37:28 -0800 Subject: [PATCH 10/11] Update submit type based on how ballot was submitted --- .../src/Controllers/Ballot/castVoteController.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/backend/src/Controllers/Ballot/castVoteController.ts b/packages/backend/src/Controllers/Ballot/castVoteController.ts index 3afb50dd..c7cbe2e0 100644 --- a/packages/backend/src/Controllers/Ballot/castVoteController.ts +++ b/packages/backend/src/Controllers/Ballot/castVoteController.ts @@ -31,9 +31,12 @@ type CastVoteEvent = { userEmail?:string, } +// NOTE: discord isn't implemented yet, but that's the plan for the future +type BallotSubmitType = 'submitted_via_browser' | 'submitted_via_admin' | 'submitted_via_discord'; + const castVoteEventQueue = "castVoteEvent"; -async function makeBallotEvent(req: IElectionRequest, targetElection: Election, inputBallot: Ballot, voter_id?: string){ +async function makeBallotEvent(req: IElectionRequest, targetElection: Election, inputBallot: Ballot, submitType: BallotSubmitType, voter_id?: string){ inputBallot.election_id = targetElection.election_id; let roll = null; @@ -74,7 +77,7 @@ async function makeBallotEvent(req: IElectionRequest, targetElection: Election, //TODO, ensure the user ID is added to the ballot... //should server-authenticate the user id based on auth token inputBallot.history.push({ - action_type:"submit", + action_type: submitType, actor: roll===null ? '' : roll.voter_id , timestamp:inputBallot.date_submitted, }); @@ -118,7 +121,7 @@ async function uploadBallotsController(req: IElectionRequest, res: Response, nex let events = await Promise.all( req.body.ballots.map(({ballot, voter_id} : {ballot: Ballot, voter_id: string}) => - makeBallotEvent(req, targetElection, structuredClone(ballot), voter_id).catch((err) => ({ + makeBallotEvent(req, targetElection, structuredClone(ballot), 'submitted_via_admin', voter_id).catch((err) => ({ error: err, ballot: ballot })) @@ -162,7 +165,7 @@ async function castVoteController(req: IElectionRequest, res: Response, next: Ne throw new BadRequest("Election is not open"); } - let event = await makeBallotEvent(req, targetElection, req.body.ballot) + let event = await makeBallotEvent(req, targetElection, req.body.ballot, 'submitted_via_browser') event.userEmail = req.body.receiptEmail; From da32685e2b7261e0a49501bf33262edc693c671b Mon Sep 17 00:00:00 2001 From: Arend Peter Date: Fri, 31 Jan 2025 12:04:53 -0800 Subject: [PATCH 11/11] Tweak migrator --- packages/backend/src/Migrations/2025_01_29_admin_upload.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/backend/src/Migrations/2025_01_29_admin_upload.ts b/packages/backend/src/Migrations/2025_01_29_admin_upload.ts index ca9c0fa7..e22a8212 100644 --- a/packages/backend/src/Migrations/2025_01_29_admin_upload.ts +++ b/packages/backend/src/Migrations/2025_01_29_admin_upload.ts @@ -6,7 +6,7 @@ export async function up(db: Kysely): Promise { live_election: Ballots submitted by voters during election prior_election: Election admin uploaded ballots from a previous election */ - .addColumn('ballot_source', 'varchar' ) + .addColumn('ballot_source', 'varchar', (col) => col.notNull()) // unique identifier for mapping public archive elections to their real elections // ex. Genola_11022021_CityCouncil .addColumn('public_archive_id', 'varchar' )