Merge pull request #14 from sanbir/new-aggregated-distributors-query

New aggregated distributors query

.env.example

@@ -25,7 +25,6 @@ KEY_FILE_NAME=service_account.json
 
 FEE_MANAGER_PROPOSERS_URL=https://eth-staking-middleware.dev-p2p.org/fee-manager/admin/proposers
 DISTRIBUTORS_URL=https://eth-staking-middleware.dev-p2p.org/deposit-manager/api/v1/distributors/active_since/1706532017137
-START_TIMESTAMP=1709279021642
 
 IS_TESTNET=true
 USE_ERC_4337=false

97-test.ts

@@ -1,38 +1,20 @@
 import "dotenv/config"
 import {logger} from "./scripts/helpers/logger";
 import {getRowsFromBigQuery} from "./scripts/getRowsFromBigQuery";
-import { PROPOSERS } from "./scripts/helpers/PROPOSERS";
 import { getFeeDistributorInputs } from "./scripts/getFeeDistributorInputs"
 import { getFdAddressesWithPeriodsFromApi } from "./scripts/getFdAddressesWithPeriodsFromApi"
+import { getLastDistributionDate } from "./scripts/helpers/getLastDistributionDate"
 
 async function main() {
     logger.info('97-test started')
 
-    const fds = await getFdAddressesWithPeriodsFromApi()
+    // const fds = await test_getRowsFromBigQuery()
 
+    const aa = await getLastDistributionDate('0x21ca9c89C704138Eb350eBF73E973E6ACd4351F8')
 
     logger.info('97-test finished')
 }
 
-
-
-async function test_manualSetup() {
-    const pubkeysWithData = PROPOSERS.proposer_config
-
-    const pubkeys = Object.keys(pubkeysWithData)
-
-    // @ts-ignore
-    const aa = pubkeys.filter(p => pubkeysWithData[p].fee_recipient === '0x0b1ddf6d1da69532ad4198470679b0b49176c68f')
-
-    const feeDistributorInputs = await getFeeDistributorInputs()
-
-    const input = feeDistributorInputs.find(f => f.fdAddress === '0x0b1Ddf6D1DA69532Ad4198470679b0b49176c68f')
-
-    const newKeys = input?.periods[0].pubkeys
-
-    const res = aa.filter(a => !newKeys?.some(n => n === a))
-}
-
 async function test_getRowsFromBigQuery() {
     const indexesWithAmounts = await getRowsFromBigQuery(
         [1217607],

scripts/getFeeDistributorInputs.ts

@@ -12,9 +12,6 @@ export async function getFeeDistributorInputs() {
     if (!process.env.REFERENCE_FEE_DISTRIBUTOR) {
         throw new Error("No REFERENCE_FEE_DISTRIBUTOR in ENV")
     }
-    if (!process.env.START_TIMESTAMP) {
-        throw new Error("No START_TIMESTAMP in ENV")
-    }
 
     const withManual = await getFdAddressesWithPeriodsFromApi()
     logger.info(Object.keys(withManual).length + ' fd addresses with periods from API with manual')
@@ -24,7 +21,6 @@ export async function getFeeDistributorInputs() {
     logger.info(fsAddresses.length + ' fd addresses with periods from API')
 
     const now = new Date()
-    const startDate = new Date(Number(process.env.START_TIMESTAMP))
 
     const feeDistributorInputs: FeeDistributorInput[] = fsAddresses.map(fdAddress => {
         const periodsFromApi = fdAddressesWithPeriodsFromApi[fdAddress]
@@ -42,9 +38,7 @@ export async function getFeeDistributorInputs() {
         }
 
         const periods: Period[] = periodsFromApi.map(pa => ({
-            startDate: new Date(pa.activated_at) < startDate
-              ? startDate
-              : new Date(pa.activated_at),
+            startDate: new Date(pa.activated_at),
 
             endDate: pa.deactivated_at
               ? new Date(pa.deactivated_at)

scripts/getValidatorWithFeeDistributorsAndAmount.ts

@@ -6,6 +6,7 @@ import fs from "fs";
 import {getFeeDistributorInputs} from "./getFeeDistributorInputs";
 import {getFeeDistributorsWithBalance} from "./getFeeDistributorsWithBalance";
 import { getFeeDistributorsWithBalanceSsv } from "./getFeeDistributorsWithBalanceSsv"
+import { getLastDistributionDate } from "./helpers/getLastDistributionDate"
 
 export async function getValidatorWithFeeDistributorsAndAmount() {
     logger.info('getValidatorWithFeeDistributorsAndAmount started')
@@ -15,6 +16,11 @@ export async function getValidatorWithFeeDistributorsAndAmount() {
     const feeDistributorsWithBalance = await getFeeDistributorsWithBalance(feeDistributorInputs)
     logger.info(feeDistributorsWithBalance.length + ' feeDistributorsWithBalance')
 
+    const filePath_feeDistributorsWithBalance = getDatedJsonFilePath('feeDistributorsWithBalance')
+    logger.info('Saving feeDistributorsWithBalance to ' + filePath_feeDistributorsWithBalance)
+    fs.writeFileSync(filePath_feeDistributorsWithBalance, JSON.stringify(feeDistributorsWithBalance))
+    logger.info('feeDistributorsWithBalance saved')
+
     const feeDistributorsWithBalanceSsv = await getFeeDistributorsWithBalanceSsv()
     logger.info(feeDistributorsWithBalanceSsv.length + ' feeDistributorsWithBalanceSsv')
 
@@ -23,6 +29,17 @@ export async function getValidatorWithFeeDistributorsAndAmount() {
     for (const fd of feeDistributorsWithBalance) {
         let fdAmount = 0
 
+        const lastDistributionDate = await getLastDistributionDate(fd.fdAddress)
+        if (lastDistributionDate) {
+            fd.periods = fd.periods.filter(p => p.endDate > lastDistributionDate)
+
+            for (const period of fd.periods) {
+                if (period.startDate < lastDistributionDate) {
+                    period.startDate = lastDistributionDate
+                }
+            }
+        }
+
         for (const period of fd.periods) {
             const pubkeys = period.pubkeys
 

scripts/helpers/getLastDistributionDate.ts

@@ -0,0 +1,34 @@
+import { logger } from "./logger"
+import { getIsContract } from "./getIsContract"
+import { getFeeDistributorContract } from "./getFeeDistributorContract"
+import { ethers } from "ethers"
+
+export async function getLastDistributionDate(fdAddress: string) : Promise<Date | null> {
+  logger.info('getLastDistributionDate started for ' + fdAddress)
+
+  const isContract = await getIsContract(fdAddress)
+  if (!isContract) {
+    return null
+  }
+
+  const fd = getFeeDistributorContract(fdAddress)
+
+  const logs = await fd.queryFilter(fd.filters.FeeDistributor__Withdrawn(), 19009010, "latest")
+
+  if (logs.length === 0) {
+    return null
+  }
+
+  const provider = new ethers.providers.JsonRpcProvider(process.env.RPC_URL)
+
+  const withdrawalDates = await Promise.all(logs.map(async (log) => {
+    const block = await provider.getBlock(log.blockNumber)
+    return new Date(block.timestamp * 1000) // Convert Unix timestamp to JavaScript Date
+  }));
+
+  const maxDate = withdrawalDates.reduce((max, date) => date > max ? date : max)
+
+  logger.info('getLastDistributionDate finished for ' + fdAddress)
+
+  return maxDate
+}

scripts/reTryWithdrawWithExistingTree.ts

@@ -13,9 +13,9 @@ export async function reTryWithdrawWithExistingTree() {
     console.log(path.dirname(process.argv[1]))
 
     // @ts-ignore
-    const fds = JSON.parse(fs.readFileSync(path.dirname(process.argv[1]) + '/reports/fee-distributors-with-legacy-already-split-amounts2024-03-01T07:43:48.600Z.json'))
+    const fds = JSON.parse(fs.readFileSync(path.dirname(process.argv[1]) + '/reports/fee-distributors-with-legacy-already-split-amounts2024-10-15T08:21:55.910Z.json'))
 
-    const merkleTreeFilePath = './reports/merkle-tree2024-03-01T07:43:48.600Z.json'
+    const merkleTreeFilePath = './reports/merkle-tree2024-10-15T08:21:55.910Z.json'
 
     // @ts-ignore
     const tree = StandardMerkleTree.load(JSON.parse(fs.readFileSync(merkleTreeFilePath)));

scripts/ssv/getSsvFeeRecipientAddressesWithTimestampsPerProxy.ts

@@ -7,7 +7,7 @@ export async function getSsvFeeRecipientAddressesWithTimestampsPerProxy(proxyAdd
 
     const ssvNetwork = getSsvNetworkContract()
 
-    const logs = await ssvNetwork.queryFilter(ssvNetwork.filters.FeeRecipientAddressUpdated(proxyAddress))
+    const logs = await ssvNetwork.queryFilter(ssvNetwork.filters.FeeRecipientAddressUpdated(proxyAddress), 19326041, "latest")
 
     const provider = new ethers.providers.JsonRpcProvider(process.env.RPC_URL)
 

scripts/ssv/getSsvPubKeysPerProxy.ts

@@ -1,17 +1,28 @@
 import {logger} from "../helpers/logger";
 import {ethers} from "ethers";
 import {getSsvNetworkContract} from "../helpers/getSsvNetworkContract";
+import { sleep } from "../helpers/sleep"
 
-export async function getSsvPubKeysPerProxy(proxyAddress: string) { // same for 3.1
+export async function getSsvPubKeysPerProxy(proxyAddress: string): Promise<string[]> { // same for 3.1
     logger.info('getSsvPubKeysPerProxy started for ' + proxyAddress)
 
     const ssvNetwork = getSsvNetworkContract()
 
-    const logs = await ssvNetwork.queryFilter(ssvNetwork.filters.ValidatorAdded(proxyAddress))
+    try {
+        const logs = await ssvNetwork.queryFilter(ssvNetwork.filters.ValidatorAdded(proxyAddress), 19326041, "latest")
 
-    const publicKeys: string[] = logs.map(log => log.args?.publicKey)
+        const publicKeys: string[] = logs.map(log => log.args?.publicKey)
 
-    logger.info('getSsvPubKeysPerProxy finished for ' + proxyAddress)
+        logger.info('getSsvPubKeysPerProxy finished for ' + proxyAddress)
 
-    return publicKeys
+        return publicKeys
+    } catch (error) {
+        logger.error(error)
+
+        logger.info('Sleeping for 5 sec...')
+        await sleep(5000)
+        logger.info('Re-trying ' + proxyAddress)
+
+        return await getSsvPubKeysPerProxy(proxyAddress)
+    }
 }