Follow on-chain storage format change

- Fold separate flags into a bigint, as bits
- Compress metadata wit CBOR
- Introduce versioned storage format for metadata
- Add proper error handling around invalid metadata

frontend/src/components/PollCard/index.tsx

@@ -1,4 +1,4 @@
-import { Proposal } from '../../types'
+import { isPollActive, Proposal } from '../../types'
 import { FC, MouseEventHandler, useEffect } from 'react'
 import { Link } from 'react-router-dom'
 import classes from './index.module.css'
@@ -94,7 +94,7 @@ export const PollCard: FC<{
 
   const {
     id: pollId,
-    proposal: { active },
+    proposal: { params },
     ipfsParams: {
       name,
       description,
@@ -111,6 +111,8 @@ export const PollCard: FC<{
     },
   }
 
+  const active = isPollActive(params)
+
   const renderedDescription = description
 
   const textMatches = searchPatterns.length

frontend/src/constants/config.ts

@@ -1,5 +1,5 @@
 // EIP-3085: wallet_addEthereumChain RPC Method
-import { ExtendedPoll } from '../types'
+import { ExtendedPoll, FLAG_ACTIVE } from '../types'
 import { randomchoice, chain_info } from '@oasisprotocol/blockvote-contracts'
 
 export const METAMASK_HOME_PAGE_URL = 'https://metamask.io/'
@@ -56,7 +56,9 @@ export const demoPoll1 = {
   id: 'demo',
   proposal: {
     id: '0xdemo',
-    active: true,
+    params: {
+      flags: FLAG_ACTIVE,
+    },
   },
   ipfsParams: {
     creator: 'demo',
@@ -73,7 +75,9 @@ export const demoPoll2 = {
   id: 'demo',
   proposal: {
     id: '0xdemo',
-    active: true,
+    params: {
+      flags: FLAG_ACTIVE,
+    },
   },
   ipfsParams: {
     creator: 'demo',
@@ -90,7 +94,9 @@ export const demoPoll3 = {
   id: 'demo',
   proposal: {
     id: '0xdemo',
-    active: true,
+    params: {
+      flags: FLAG_ACTIVE,
+    },
   },
   ipfsParams: {
     creator: 'demo',
@@ -107,7 +113,9 @@ export const demoPoll4 = {
   id: 'demo',
   proposal: {
     id: '0xdemo',
-    active: true,
+    params: {
+      flags: FLAG_ACTIVE,
+    },
   },
   ipfsParams: {
     creator: 'demo',

frontend/src/hooks/useExtendedPoll.ts

@@ -1,11 +1,22 @@
-import { ListOfVotes, ExtendedPoll, PollResults, Proposal, ListOfVoters } from '../types'
+import {
+  ListOfVotes,
+  ExtendedPoll,
+  PollResults,
+  Proposal,
+  ListOfVoters,
+  isPollActive,
+  shouldPublishVotes,
+  shouldPublishVoters,
+  inactivatePoll,
+  Poll,
+} from '../types'
 import { useEffect, useMemo, useState } from 'react'
 import { dashboard, demoSettings, getDemoPoll } from '../constants/config'
 import { usePollGaslessStatus } from './usePollGaslessStatus'
 import { usePollPermissions } from './usePollPermissions'
 import { useEthereum } from './useEthereum'
 import { useContracts } from './useContracts'
-import { decodeBase64, toUtf8String } from 'ethers'
+import { decodePollMetadata } from '../utils/poll.utils'
 import { getVerdict } from '../components/InputFields'
 
 const noVoters: ListOfVoters = { out_count: 0n, out_voters: [] }
@@ -36,10 +47,14 @@ export const useExtendedPoll = (
 
   let correctiveAction: (() => void) | undefined
 
-  const ipfsParams = useMemo(
-    () => (metadata ? JSON.parse(toUtf8String(decodeBase64(metadata))) : undefined),
-    [metadata],
-  )
+  const [ipfsParams, ipfsError] = useMemo((): [Poll | undefined, string | undefined] => {
+    try {
+      return metadata ? [decodePollMetadata(metadata), undefined] : [undefined, undefined]
+    } catch (e) {
+      console.log('metadata problem on poll', proposal?.id, e)
+      return [undefined, "Invalid metadata, poll can't be displayed."]
+    }
+  }, [metadata])
 
   useEffect(
     // Update poll object
@@ -77,10 +92,10 @@ export const useExtendedPoll = (
     correctiveAction = checkPermissions
   }
 
-  const isActive = !!proposal?.active
+  const isActive = isPollActive(proposal?.params)
 
   const loadVotes = async () => {
-    if (isDemo || !proposal || !pollManager || !ipfsParams || proposal.active) return
+    if (isDemo || !proposal || !pollManager || !ipfsParams || isActive) return
 
     if (params.onDashboard && !dashboard.showResults) return
 
@@ -90,7 +105,7 @@ export const useExtendedPoll = (
     setVoteCounts(voteCounts)
     setWinningChoice(proposal.topChoice)
 
-    if (proposal.params.publishVotes) {
+    if (shouldPublishVotes(proposal.params)) {
       const loadedVotes: ListOfVotes = {
         out_count: 1000n, // Fake number, will be updated when the first batch is loaded
         out_voters: [],
@@ -103,7 +118,7 @@ export const useExtendedPoll = (
         loadedVotes.out_choices.push(...newVotes.out_choices)
       }
       setVotes(loadedVotes)
-    } else if (proposal.params.publishVoters) {
+    } else if (shouldPublishVoters(proposal.params)) {
       const loadedVoters: ListOfVoters = {
         out_count: 1000n, // Fake number, will be updated when the first batch is loaded
         out_voters: [],
@@ -124,7 +139,7 @@ export const useExtendedPoll = (
   useEffect(
     // Load votes, when the stars are right
     () => void loadVotes(),
-    [proposal, proposal?.active, params.onDashboard, dashboard.showResults, pollManager, ipfsParams, isDemo],
+    [proposal, isActive, params.onDashboard, dashboard.showResults, pollManager, ipfsParams, isDemo],
   )
 
   const pollResults = useMemo(() => {
@@ -155,7 +170,7 @@ export const useExtendedPoll = (
       ...poll,
       proposal: {
         ...poll.proposal,
-        active: false,
+        params: inactivatePoll(poll.proposal.params),
         topChoice: 0n,
       },
     })
@@ -178,7 +193,7 @@ export const useExtendedPoll = (
     isActive,
     isDemo,
     isLoading: false,
-    error: undefined,
+    error: ipfsError,
     correctiveAction,
     poll,
     voteCounts,

frontend/src/hooks/useProposalFromChain.ts

@@ -30,7 +30,7 @@ export const useProposalFromChain = (proposalId: string) => {
     try {
       setIsLoading(true)
       const data = await pollManager.PROPOSALS(proposalId)
-      const [active, topChoice, params] = data
+      const [topChoice, params] = data
       const acl = data?.params?.acl
       if (!acl || acl === ZeroAddress) {
         // setError('Found proposal with invalid ACL.')
@@ -39,7 +39,6 @@ export const useProposalFromChain = (proposalId: string) => {
       } else {
         setProposal({
           id: proposalId,
-          active,
           topChoice,
           params,
         })

frontend/src/pages/DashboardPage/useDashboardData.ts

@@ -1,6 +1,6 @@
 import { useContracts } from '../../hooks/useContracts'
 import { useEffect, useMemo, useState } from 'react'
-import { PollManager, Proposal } from '../../types'
+import { isPollActive, PollManager, Proposal } from '../../types'
 import { useEthereum } from '../../hooks/useEthereum'
 import { useBooleanField, useOneOfField, useTextField } from '../../components/InputFields'
 import { useNavigate } from 'react-router-dom'
@@ -22,9 +22,9 @@ export type WantedStatus = 'active' | 'completed' | 'all'
 export const isPollStatusAcceptable = (proposal: Proposal, wantedStatus: WantedStatus): boolean => {
   switch (wantedStatus) {
     case 'active':
-      return proposal.active
+      return isPollActive(proposal.params)
     case 'completed':
-      return !proposal.active
+      return !isPollActive(proposal.params)
     case 'all':
       return true
   }
@@ -135,8 +135,8 @@ async function fetchProposals(
     }
 
     result.out_proposals.forEach(({ id, proposal }) => {
-      const [active, topChoice, params] = proposal
-      proposalList.push({ id, active, topChoice, params })
+      const [topChoice, params] = proposal
+      proposalList.push({ id, topChoice, params })
     })
 
     if (result.out_proposals.length < FETCH_BATCH_SIZE) {
@@ -204,10 +204,7 @@ export const useDashboardData = () => {
   }
 
   const allProposals = useMemo(
-    () => [
-      ...activeProposals.map((p): Proposal => ({ ...p, active: true })),
-      ...pastProposals.map((p): Proposal => ({ ...p, active: false })),
-    ],
+    () => [...activeProposals, ...pastProposals],
     [activeProposals, pastProposals, userAddress],
   )
 

frontend/src/pages/PollPage/hook.ts

@@ -18,6 +18,7 @@ import { getVerdict } from '../../components/InputFields'
 import { useExtendedPoll } from '../../hooks/useExtendedPoll'
 import { useProposalFromChain } from '../../hooks/useProposalFromChain'
 import { useNavigate } from 'react-router-dom'
+import { isPollActive } from '../../types'
 
 export const usePollData = (pollId: string) => {
   const navigate = useNavigate()
@@ -362,7 +363,7 @@ export const usePollData = (pollId: string) => {
     () => {
       if (
         isDemo &&
-        poll?.proposal.active &&
+        isPollActive(poll?.proposal.params) &&
         remainingTime?.isPastDue &&
         remainingTime.totalSeconds < demoSettings.waitSecondsBeforeFormallyCompleting + 5 &&
         remainingTime.totalSeconds >= demoSettings.waitSecondsBeforeFormallyCompleting
@@ -380,7 +381,7 @@ export const usePollData = (pollId: string) => {
     if (hasCompleted) {
       if (!poll) {
         // console.log("No poll loaded, waiting to load")
-      } else if (poll.proposal.active) {
+      } else if (isPollActive(poll.proposal.params)) {
         // console.log("Apparently, we have completed a poll, but we still perceive it as active, so scheduling a reload...")
         setTimeout(() => {
           // console.log("Reloading now")
@@ -400,7 +401,7 @@ export const usePollData = (pollId: string) => {
     isLoading: isProposalLoading || isLoading,
     error: proposalError ?? error,
     poll,
-    active: !!poll?.proposal?.active,
+    active: isPollActive(poll?.proposal?.params),
 
     selectedChoice: winningChoice ?? selectedChoice,
     canSelect,

frontend/src/types/index.ts

@@ -1,4 +1,5 @@
 export type { DataEntry } from './data-entry'
 export type * from './poll'
+export * from './poll-flags'
 export type { IconSize } from './icon-size'
 export type { IconProps } from './icon-props'

frontend/src/types/poll-flags.ts

@@ -0,0 +1,23 @@
+import { PollManager } from './poll'
+
+export const FLAG_ACTIVE = 1n << 0n
+export const FLAG_PUBLISH_VOTERS = 1n << 1n
+export const FLAG_PUBLISH_VOTES = 1n << 2n
+export const FLAG_HIDDEN = 1n << 3n
+export const FLAG_WEIGHT_LOG10 = 1n << 4n
+export const FLAG_WEIGHT_ONE = 1n << 5n
+
+type Params = PollManager.ProposalParamsStructOutput
+
+export const isPollActive = (params: Params | undefined): boolean => !!((params?.flags ?? 0n) & FLAG_ACTIVE)
+
+export const inactivatePoll = (params: Params): Params => ({
+  ...params,
+  flags: isPollActive(params) ? params.flags ^ FLAG_ACTIVE : params.flags,
+})
+
+export const shouldPublishVotes = (params: Params | undefined): boolean =>
+  !!((params?.flags ?? 0n) & FLAG_PUBLISH_VOTES)
+
+export const shouldPublishVoters = (params: Params | undefined): boolean =>
+  !!((params?.flags ?? 0n) & FLAG_PUBLISH_VOTERS)

frontend/src/types/poll.ts

@@ -12,7 +12,6 @@ export type {
 
 export type Proposal = {
   id: string
-  active: boolean
   topChoice: bigint
   params: PollManager.ProposalParamsStructOutput
 }

frontend/src/utils/poll.utils.ts

@@ -1,12 +1,6 @@
-import {
-  AbiCoder,
-  BytesLike,
-  encodeBase64,
-  getAddress,
-  JsonRpcProvider,
-  ParamType,
-  toUtf8Bytes,
-} from 'ethers'
+import { AbiCoder, BytesLike, getAddress, JsonRpcProvider, ParamType } from 'ethers'
+
+import { encode as cborEncode, decode as cborDecode } from 'cbor-web'
 
 import {
   chain_info,
@@ -25,7 +19,7 @@ import {
 } from '@oasisprotocol/blockvote-contracts'
 export type { ContractType, NftType } from '@oasisprotocol/blockvote-contracts'
 export { isToken } from '@oasisprotocol/blockvote-contracts'
-import { Poll, PollManager } from '../types'
+import { FLAG_HIDDEN, FLAG_PUBLISH_VOTERS, FLAG_PUBLISH_VOTES, Poll, PollManager } from '../types'
 import { EthereumContext } from '../providers/EthereumContext'
 import { DecisionWithReason, denyWithReason } from '../components/InputFields'
 import { FetcherFetchOptions } from './StoredLRUCache'
@@ -141,6 +135,27 @@ export type CreatePollProps = {
   completionTime: Date | undefined
 }
 
+const CURRENT_ENCODING_VERSION = 0
+
+const encodePollMetadata = (poll: Poll): Uint8Array => {
+  const encoded = cborEncode({ v: CURRENT_ENCODING_VERSION, data: poll })
+  // console.log('Encoded poll data', encoded)
+  return encoded
+}
+
+export const decodePollMetadata = (metadata: string): Poll => {
+  const { v, data } = cborDecode(metadata.substring(2), { preferWeb: true, encoding: 'hex' })
+
+  if (typeof v !== 'number') throw new Error('Unknown poll data format')
+
+  switch (v as number) {
+    case CURRENT_ENCODING_VERSION:
+      return data as Poll
+    default:
+      throw new Error(`Unrecognized poll data format version: ${v}`)
+  }
+}
+
 export const createPoll = async (
   pollManager: PollManager,
   creator: string,
@@ -176,14 +191,18 @@ export const createPoll = async (
 
   // console.log('Compiling poll', poll)
 
+  let pollFlags: bigint = 0n
+
+  if (poll.options.publishVoters) pollFlags |= FLAG_PUBLISH_VOTERS
+  if (poll.options.publishVotes) pollFlags |= FLAG_PUBLISH_VOTES
+  if (isHidden) pollFlags |= FLAG_HIDDEN
+
   const proposalParams: PollManager.ProposalParamsStruct = {
-    metadata: encodeBase64(toUtf8Bytes(JSON.stringify(poll))),
+    metadata: encodePollMetadata(poll),
     numChoices: answers.length,
-    publishVotes: poll.options.publishVotes,
-    publishVoters: poll.options.publishVoters,
     closeTimestamp: poll.options.closeTimestamp,
     acl: aclOptions.address,
-    isHidden,
+    flags: pollFlags,
   }
 
   console.log('params are', proposalParams)