Merge pull request #312 from EYBlockchain/ilyas/optimist-depends

Remove Optimist dependency in client

cli/lib/nf3.mjs

@@ -360,14 +360,8 @@ class Nf3 {
   */
   async finaliseWithdrawal(withdrawTransactionHash) {
     // find the L2 block containing the L2 transaction hash
-    let res = await axios.get(
-      `${this.optimistBaseUrl}/block/transaction-hash/${withdrawTransactionHash}`,
-    );
-    const { block, transactions, index } = res.data;
-    res = await axios.post(`${this.clientBaseUrl}/finalise-withdrawal`, {
-      block,
-      transactions,
-      index,
+    const res = await axios.post(`${this.clientBaseUrl}/finalise-withdrawal`, {
+      transactionHash: withdrawTransactionHash,
     });
     return this.submitTransaction(res.data.txDataToSign, this.shieldContractAddress, 0);
   }
@@ -381,16 +375,9 @@ class Nf3 {
   @param {number} fee - the amount being paid for the instant withdrawal service
   */
   async requestInstantWithdrawal(withdrawTransactionHash, fee) {
-    // find the L2 block containing the L2 transaction hash
-    let res = await axios.get(
-      `${this.optimistBaseUrl}/block/transaction-hash/${withdrawTransactionHash}`,
-    );
-    const { block, transactions, index } = res.data;
     // set the instant withdrawal fee
-    res = await axios.post(`${this.clientBaseUrl}/set-instant-withdrawal`, {
-      block,
-      transactions,
-      index,
+    const res = await axios.post(`${this.clientBaseUrl}/set-instant-withdrawal`, {
+      transactionHash: withdrawTransactionHash,
     });
     return this.submitTransaction(res.data.txDataToSign, this.shieldContractAddress, fee);
   }

nightfall-optimist/src/classes/fq2.mjs → common-files/classes/fq2.mjs

@@ -3,8 +3,9 @@ Class representing a complex modular number.  In particular, this is useful
 for dealing with f_q^2 field elements in the alt BN128 curve.
 */
 
+// eslint-disable-next-line import/no-extraneous-dependencies
 import config from 'config';
-import { modDivide, complexDivMod } from '../utils/curve-maths/modular-division.mjs';
+import { modDivide, complexDivMod } from '../utils/crypto/modular-division.mjs';
 
 const { BN128_PRIME_FIELD } = config;
 

nightfall-optimist/src/utils/curve-maths/modular-division.mjs → common-files/utils/crypto/modular-division.mjs

@@ -1,6 +1,7 @@
 // modular division
+// eslint-disable-next-line import/no-extraneous-dependencies
 import config from 'config';
-import { mulMod, addMod } from 'common-files/utils/crypto/number-theory.mjs';
+import { mulMod, addMod } from './number-theory.mjs';
 
 const { BN128_PRIME_FIELD } = config;
 

common-files/utils/curve-maths/curves.mjs

@@ -5,8 +5,14 @@ module for manupulating elliptic curve points for an alt-bn128 curve. This
 is the curve that Ethereum currently has pairing precompiles for. All the
 return values are BigInts (or arrays of BigInts).
 */
+// eslint-disable-next-line import/no-extraneous-dependencies
+import config from 'config';
+import { mulMod, addMod, squareRootModPrime } from '../crypto/number-theory.mjs';
+import Fq2 from '../../classes/fq2.mjs';
 import Proof from '../../classes/proof.mjs';
 
+const { BN128_PRIME_FIELD } = config;
+
 /**
 function to compress a G1 point. If we throw away the y coodinate, we can
 recover it using the curve equation later, and save almost half the storage.
@@ -50,3 +56,52 @@ export function compressProof(_proof) {
   const compressed = [compressG1(proof.a), compressG2(proof.b), compressG1(proof.c)];
   return compressed.flat();
 }
+
+/**
+solving Y^2 = X^3 + 3 over p
+*/
+export function decompressG1(xin) {
+  // first, extract the parity bit
+  const xbin = BigInt(xin).toString(2).padStart(256, '0');
+  const parity = xbin[0];
+  // then convert the rest into a BigInt
+  const x = BigInt(`0b${xbin.slice(1)}`);
+  const x3 = mulMod([x, x, x], BN128_PRIME_FIELD);
+  const y2 = addMod([x3, 3n], BN128_PRIME_FIELD);
+  let y = squareRootModPrime(y2, BN128_PRIME_FIELD);
+  if (parity !== y.toString(2).slice(-1)) y = BN128_PRIME_FIELD - y;
+  return [`0x${x.toString(16).padStart(64, '0')}`, `0x${y.toString(16).padStart(64, '0')}`];
+}
+
+/**
+solving Y^2 = X^3 + 3/(i+9)
+*/
+export function decompressG2(xin) {
+  // first extract parity bits
+  const xbin = xin.map(c => BigInt(c).toString(2).padStart(256, '0'));
+  const parity = xbin.map(xb => xb[0]); // extract parity
+  const x = new Fq2(...xbin.map(xb => BigInt(`0b${xb.slice(1)}`))); // x element
+  const x3 = x.mul(x).mul(x);
+  const d = new Fq2(3n, 0n).div(new Fq2(9n, 1n)); // TODO hardcode this?
+  const y2 = x3.add(d);
+  const y = y2.sqrt();
+  // fix the parity of y
+  const a = parity[0] === y.real.toString(2).slice(-1) ? y.real : BN128_PRIME_FIELD - y.real;
+  const b =
+    parity[1] === y.imaginary.toString(2).slice(-1) ? y.imaginary : BN128_PRIME_FIELD - y.imaginary;
+  // we return arrays of real and imaginary points not Fq2.
+  return [x.toHex(), new Fq2(a, b).toHex()];
+}
+
+export function decompressProof(compressedProof) {
+  // compressed proofs are always just flat arrays. We also return a flattend
+  // proof array as we rarely need the object.  If we do we can construct one
+  // from the flattened array as an instance of the Proof class. This returns
+  // and array of promises so be sure to await Promise.all.
+  const [aCompressed, bCompressedReal, bCompressedImaginary, cCompressed] = compressedProof;
+  return [
+    decompressG1(aCompressed),
+    decompressG2([bCompressedReal, bCompressedImaginary]),
+    decompressG1(cCompressed),
+  ].flat(2);
+}

nightfall-client/src/event-handlers/block-proposed.mjs

@@ -11,7 +11,7 @@ import {
 import getProposeBlockCalldata from '../services/process-calldata.mjs';
 import Secrets from '../classes/secrets.mjs';
 import { ivks, nsks } from '../services/keys.mjs';
-import { getLatestTree, saveTree } from '../services/database.mjs';
+import { getLatestTree, saveTree, saveTransaction, saveBlock } from '../services/database.mjs';
 
 const { ZERO } = config;
 
@@ -21,9 +21,16 @@ This handler runs whenever a BlockProposed event is emitted by the blockchain
 async function blockProposedEventHandler(data) {
   logger.info(`Received Block Proposed event`);
   // ivk will be used to decrypt secrets whilst nsk will be used to calculate nullifiers for commitments and store them
-  const { transactions, blockNumberL2 } = await getProposeBlockCalldata(data);
+  const { blockNumber: currentBlockCount, transactionHash: transactionHashL1 } = data;
+  const { transactions, block } = await getProposeBlockCalldata(data);
   const latestTree = await getLatestTree();
   const blockCommitments = transactions.map(t => t.commitments.filter(c => c !== ZERO)).flat();
+
+  if ((await countCommitments(blockCommitments)) > 0) {
+    await saveBlock({ blockNumber: currentBlockCount, transactionHashL1, ...block });
+    await Promise.all(transactions.map(t => saveTransaction({ transactionHashL1, ...t })));
+  }
+
   const dbUpdates = transactions.map(async transaction => {
     // filter out non zero commitments and nullifiers
     const nonZeroCommitments = transaction.commitments.flat().filter(n => n !== ZERO);
@@ -54,10 +61,10 @@ async function blockProposedEventHandler(data) {
     }
     return [
       Promise.all(storeCommitments),
-      markOnChain(nonZeroCommitments, blockNumberL2, data.blockNumber, data.transactionHash),
+      markOnChain(nonZeroCommitments, block.blockNumberL2, data.blockNumber, data.transactionHash),
       markNullifiedOnChain(
         nonZeroNullifiers,
-        blockNumberL2,
+        block.blockNumberL2,
         data.blockNumber,
         data.transactionHash,
       ),
@@ -67,7 +74,7 @@ async function blockProposedEventHandler(data) {
   // await Promise.all(toStore);
   await Promise.all(dbUpdates);
   const updatedTimber = Timber.statelessUpdate(latestTree, blockCommitments);
-  await saveTree(data.blockNumber, blockNumberL2, updatedTimber);
+  await saveTree(data.blockNumber, block.blockNumberL2, updatedTimber);
 
   await Promise.all(
     // eslint-disable-next-line consistent-return

nightfall-client/src/event-handlers/rollback.mjs

@@ -11,14 +11,20 @@ import {
   deleteCommitments,
   getCommitmentsFromBlockNumberL2,
 } from '../services/commitment-storage.mjs';
-import { deleteTreeByBlockNumberL2 } from '../services/database.mjs';
+import {
+  deleteTreeByBlockNumberL2,
+  deleteBlocksByBlockNumberL2,
+  findBlocksFromBlockNumberL2,
+  deleteTransactionsByTransactionHashes,
+} from '../services/database.mjs';
 
 async function rollbackEventHandler(data) {
   const { blockNumberL2 } = data.returnValues;
   logger.info(`Received Rollback event, with layer 2 block number ${blockNumberL2}`);
 
   // We get the commitments from blockNumberL2 + 1 because the bad block itself (except
   // the reason it is bad) contains valid transactions, we should not drop these.
+  // If we clear the commitments in blockNumberL2, we may spend them again while they are in an optimist mempool.
   const commitments = await getCommitmentsFromBlockNumberL2(Number(blockNumberL2) + 1);
   // Deposit transactions should not be dropped because they are always valid even post-rollback.
   const nonDeposit = commitments.filter(t => t.transactionType !== '0').map(c => c._id);
@@ -30,9 +36,14 @@ async function rollbackEventHandler(data) {
   const cResult = await clearOnChain(Number(blockNumberL2));
   logger.debug(`Rollback moved ${cResult.result.nModified} commitments off-chain`);
 
+  const blocksToDelete = await findBlocksFromBlockNumberL2(Number(blockNumberL2));
+  const txsToDelete = blocksToDelete.map(b => b.transactionHashes).flat(Infinity);
+
   await Promise.all([
-    deleteTreeByBlockNumberL2(Number(Number(blockNumberL2))),
+    deleteTreeByBlockNumberL2(Number(blockNumberL2)),
     deleteCommitments(nonDeposit),
+    deleteBlocksByBlockNumberL2(Number(blockNumberL2)),
+    deleteTransactionsByTransactionHashes(txsToDelete),
   ]);
 }
 

nightfall-client/src/services/commitment-storage.mjs

@@ -8,8 +8,8 @@ import gen from 'general-number';
 import mongo from 'common-files/utils/mongo.mjs';
 import logger from 'common-files/utils/logger.mjs';
 import { Commitment, Nullifier } from '../classes/index.mjs';
-import { getBlockByTransactionHash } from '../utils/optimist.mjs';
 import { isValidWithdrawal } from './valid-withdrawal.mjs';
+import { getBlockByBlockNumberL2, getTransactionByTransactionHash } from './database.mjs';
 
 const { MONGO_URL, COMMITMENTS_DB, COMMITMENTS_COLLECTION } = config;
 const { generalise } = gen;
@@ -33,7 +33,6 @@ export async function storeCommitment(commitment, nsk) {
     isNullifiedOnChain: Number(commitment.isNullifiedOnChain) || -1,
     nullifier: nullifierHash,
     blockNumber: -1,
-    transactionNullified: null,
   };
   // a chain reorg may cause an attempted overwrite. We should allow this, hence
   // the use of replaceOne.
@@ -95,7 +94,12 @@ export async function markNullified(commitment, transaction) {
   const connection = await mongo.connection(MONGO_URL);
   const query = { _id: commitment.hash.hex(32) };
   const update = {
-    $set: { isPendingNullification: false, isNullified: true, transactionNullified: transaction },
+    $set: {
+      isPendingNullification: false,
+      isNullified: true,
+      nullifierTransactionType: BigInt(transaction.transactionType).toString(),
+      transactionHash: transaction.transactionHash,
+    },
   };
   const db = connection.db(COMMITMENTS_DB);
   return db.collection(COMMITMENTS_COLLECTION).updateOne(query, update);
@@ -218,7 +222,7 @@ export async function getWalletBalance() {
   // commitment balances to get a balance for each erc address.
   return wallet
     .map(e => ({
-      ercAddress: BigInt(e.preimage.ercAddress).toString(16),
+      ercAddress: `0x${BigInt(e.preimage.ercAddress).toString(16).padStart(40, '0')}`, // Pad this to actual address length
       compressedPkd: e.preimage.compressedPkd,
       tokenId: !!BigInt(e.preimage.tokenId),
       value: Number(BigInt(e.preimage.value)),
@@ -258,7 +262,7 @@ export async function getWalletCommitments() {
   // commitment balances to get a balance for each erc address.
   return wallet
     .map(e => ({
-      ercAddress: BigInt(e.preimage.ercAddress).toString(16),
+      ercAddress: `0x${BigInt(e.preimage.ercAddress).toString(16).padStart(40, '0')}`,
       compressedPkd: e.preimage.compressedPkd,
       tokenId: !!BigInt(e.preimage.tokenId),
       value: Number(BigInt(e.preimage.value)),
@@ -283,55 +287,41 @@ export async function getWithdrawCommitments() {
   const db = connection.db(COMMITMENTS_DB);
   const query = {
     isNullified: true,
-    'transactionNullified.transactionType':
-      '0x0000000000000000000000000000000000000000000000000000000000000003',
+    nullifierTransactionType: '3',
+    isNullifiedOnChain: { $gte: 0 },
   };
-  const options = {
-    projection: {
-      preimage: { ercAddress: 1, compressedPkd: 1, tokenId: 1, value: 1 },
-      _id: 0,
-      transactionNullified: 1,
-    },
-  };
-  const withdraws = await db.collection(COMMITMENTS_COLLECTION).find(query, options).toArray();
-  const withdrawsDetails = withdraws
-    .map(e => ({
-      ercAddress: BigInt(e.preimage.ercAddress).toString(16),
-      compressedPkd: e.preimage.compressedPkd,
-      tokenId: !!BigInt(e.preimage.tokenId),
-      value: Number(BigInt(e.preimage.value)),
-      transactionNullified: e.transactionNullified,
-    }))
-    .filter(e => e.tokenId || e.value > 0) // there should be no commitments with tokenId and value of ZERO
-    .map(e => ({
-      compressedPkd: e.compressedPkd,
-      ercAddress: e.ercAddress,
-      balance: e.tokenId ? 1 : e.value,
-      transactionNullified: e.transactionNullified,
-    }));
+  // Get associated nullifiers of commitments that have been spent on-chain and are used for withdrawals.
+  const withdraws = await db.collection(COMMITMENTS_COLLECTION).find(query).toArray();
 
-  const withdrawsDetailsValid = await Promise.all(
-    withdrawsDetails.map(async e => {
-      let valid = false;
+  // To check validity we need the withdrawal transaction, the block the transaction is in and all other
+  // transactions in the block. We need this for on-chain validity checks.
+  const blockTxs = await Promise.all(
+    withdraws.map(async w => {
+      const block = await getBlockByBlockNumberL2(w.isNullifiedOnChain);
+      const transactions = await Promise.all(
+        block.transactionHashes.map(t => getTransactionByTransactionHash(t)),
+      );
+      const index = block.transactionHashes.findIndex(t => t === w.transactionHash);
+      return {
+        block,
+        transactions,
+        index,
+        compressedPkd: w.preimage.compressedPkd,
+        ercAddress: `0x${BigInt(w.preimage.ercAddress).toString(16).padStart(40, '0')}`, // Pad this to be a correct address length
+        balance: w.preimage.tokenId ? 1 : w.preimage.value,
+      };
+    }),
+  );
 
-      try {
-        const { block, transactions, index } = await getBlockByTransactionHash(
-          e.transactionNullified.transactionHash,
-        );
-        try {
-          const res = await isValidWithdrawal({ block, transactions, index });
-          valid = res;
-        } catch (ex) {
-          valid = false;
-        }
-      } catch (ex) {
-        valid = false;
-      }
+  // Run the validity check for each of the potential withdraws we have.
+  const withdrawsDetailsValid = await Promise.all(
+    blockTxs.map(async wt => {
+      const { block, transactions, index } = wt;
+      const valid = await isValidWithdrawal({ block, transactions, index });
       return {
-        compressedPkd: e.compressedPkd,
-        ercAddress: e.ercAddress,
-        balance: e.balance,
-        transactionNullified: e.transactionNullified,
+        compressedPkd: wt.compressedPkd,
+        ercAddress: wt.ercAddress,
+        balance: wt.balance,
         valid,
       };
     }),