Merge pull request #15 from p2p-org/new-aggregated-distributors-query

New aggregated distributors query

.env.example

@@ -2,12 +2,18 @@ PRIVATE_KEY=<YOUR_PRIVATE_KEY>
 RPC_URL=https://goerli.infura.io/v3/<YOUR_INFURA_KEY>
 
 ORACLE_ADDRESS=0xc62f6020283347EcE3D041571d8f4eF04304d506
+
 FACTORY_ADDRESS=0x22CE48f95bD232F4840f6eCF825C957924735E29
+FACTORY_3_1_ADDRESS=0xecA6e48C44C7c0cAf4651E5c5089e564031E8b90
+FACTORY_3_1_SSV_ADDRESS=0xa36420834ddd3a45F75016a328C784B2EE77e136
 
 P2P_SSV_PROXY_FACTORY_ADDRESS=0x56576C02A343c524a9C26fC3734233Ef051d529e
+P2P_SSV_PROXY_FACTORY_ADDRESS_3_1=0xcb924D4BE3Ff04B2d2116fE116138950373111d9
 SSV_NETWORK_ADDRESS=0xC3CD9A0aE89Fff83b71b58b6512D43F8a41f363D
 
-REFERENCE_FEE_DISTRIBUTOR=0x6bb18EB3FbFF556d8b02E8eaDc5F51f21436Ec79
+REFERENCE_FEE_DISTRIBUTOR=0x7109DeEb07aa9Eed1e2613F88b2f3E1e6C05163f
+REFERENCE_FEE_DISTRIBUTOR_3_1=0xca5309DAdE84EBDc71892B3de39748b7621c3760
+REFERENCE_FEE_DISTRIBUTORY_3_1_SSV=0x02AC591Ab7EF9dCa9d3044237944B7110239d778
 
 MAX_FEE_PER_GAS=50000000000
 MAX_PIORITY_FEE_PER_GAS=1000000000
@@ -19,9 +25,6 @@ KEY_FILE_NAME=service_account.json
 
 FEE_MANAGER_PROPOSERS_URL=https://eth-staking-middleware.dev-p2p.org/fee-manager/admin/proposers
 DISTRIBUTORS_URL=https://eth-staking-middleware.dev-p2p.org/deposit-manager/api/v1/distributors/active_since/1706532017137
-SESSION_PUBKEYS_URL=https://eth-staking-middleware.p2p.org/account-manager/admin/accounts/validators/query?staking_session_id=
-SESSION_PUBKEYS_USERNAME=<YOUR_SESSION_PUBKEYS_USERNAME>
-SESSION_PUBKEYS_PASSWORD=<YOUR_SESSION_PUBKEYS_PASSWORD>
 
 IS_TESTNET=true
 USE_ERC_4337=false

02-fee-distributors-with-amounts-from-db.ts

@@ -1,11 +1,11 @@
 import "dotenv/config"
 import {logger} from "./scripts/helpers/logger";
-import {getFeeDistributorsWithAmountsFromDb} from "./scripts/getFeeDistributorsWithAmountsFromDb";
+import { getValidatorWithFeeDistributorsAndAmount } from "./scripts/getValidatorWithFeeDistributorsAndAmount"
 
 async function main() {
     logger.info('02-fee-distributors-with-amounts-from-db started')
 
-    await getFeeDistributorsWithAmountsFromDb()
+    await getValidatorWithFeeDistributorsAndAmount()
 
     logger.info('02-fee-distributors-with-amounts-from-db finished')
 }

04-balances.ts

@@ -10,7 +10,7 @@ async function main() {
 
     const fds = await getFeeDistributorsWithUpdatedAmountsFromAlreadySplitClRewards()
 
-    const feeDistributorsAddresses = fds.map(fd => fd.feeDistributor)
+    const feeDistributorsAddresses = fds.map(fd => fd.fdAddress)
 
     await getAllBalances(feeDistributorsAddresses, 'balances-before')
 

05-merkle-tree.ts

@@ -1,19 +1,17 @@
 import "dotenv/config"
 import {buildMerkleTreeForFeeDistributorAddress} from "./scripts/helpers/buildMerkleTreeForFeeDistributorAddress";
 import {logger} from "./scripts/helpers/logger";
-import fs from "fs";
 import {
     getFeeDistributorsWithUpdatedAmountsFromAlreadySplitClRewards
 } from "./scripts/getFeeDistributorsWithUpdatedAmountsFromAlreadySplitClRewards";
-import {getDatedJsonFilePath} from "./scripts/helpers/getDatedJsonFilePath";
 
 async function main() {
     logger.info('05-merkle-tree started')
 
     const fds = await getFeeDistributorsWithUpdatedAmountsFromAlreadySplitClRewards()
 
     const rewardData = fds.map(fd => {
-        return [fd.feeDistributor, fd.amount.toString()]
+        return [fd.fdAddress, fd.amount.toString()]
     })
 
     buildMerkleTreeForFeeDistributorAddress(rewardData)

06-report-root-to-oracle-contract.ts

@@ -1,20 +1,18 @@
 import "dotenv/config"
 import {buildMerkleTreeForFeeDistributorAddress} from "./scripts/helpers/buildMerkleTreeForFeeDistributorAddress";
 import {logger} from "./scripts/helpers/logger";
-import fs from "fs";
 import {makeOracleReport} from "./scripts/makeOracleReport";
 import {
     getFeeDistributorsWithUpdatedAmountsFromAlreadySplitClRewards
 } from "./scripts/getFeeDistributorsWithUpdatedAmountsFromAlreadySplitClRewards";
-import {getDatedJsonFilePath} from "./scripts/helpers/getDatedJsonFilePath";
 
 async function main() {
     logger.info('06-report-root-to-oracle-contract started')
 
     const fds = await getFeeDistributorsWithUpdatedAmountsFromAlreadySplitClRewards()
 
     const rewardData = fds.map(fd => {
-        return [fd.feeDistributor, fd.amount.toString()]
+        return [fd.fdAddress, fd.amount.toString()]
     })
 
     const tree = buildMerkleTreeForFeeDistributorAddress(rewardData)

97-test.ts

@@ -1,19 +1,26 @@
 import "dotenv/config"
 import {logger} from "./scripts/helpers/logger";
-import {reTryWithdrawWithExistingTree} from "./scripts/reTryWithdrawWithExistingTree";
 import {getRowsFromBigQuery} from "./scripts/getRowsFromBigQuery";
+import { getFeeDistributorInputs } from "./scripts/getFeeDistributorInputs"
+import { getFdAddressesWithPeriodsFromApi } from "./scripts/getFdAddressesWithPeriodsFromApi"
+import { getLastDistributionDate } from "./scripts/helpers/getLastDistributionDate"
 
 async function main() {
     logger.info('97-test started')
 
+    // const fds = await test_getRowsFromBigQuery()
+
+    const aa = await getLastDistributionDate('0x21ca9c89C704138Eb350eBF73E973E6ACd4351F8')
+
+    logger.info('97-test finished')
+}
+
+async function test_getRowsFromBigQuery() {
     const indexesWithAmounts = await getRowsFromBigQuery(
         [1217607],
         new Date('2024-02-01'),
         new Date('2024-03-01')
     )
-
-
-    logger.info('97-test finished')
 }
 
 // We recommend this pattern to be able to use async/await everywhere

scripts/deployFeeDistributor.ts

@@ -1,15 +1,18 @@
-import {
-    getFeeDistributorFactoryContractSigned
-} from "./helpers/getFeeDistributorFactoryContract";
-import {FeeDistributorIdentityParams} from "./models/FeeDistributorIdentityParams";
 import {ContractTransaction} from "ethers";
 import {getNonce, incrementNonce} from "./helpers/nonce";
+import { getMatchingFactory } from "./helpers/getMatchingFactory"
+import { FeeDistributorInput } from "./models/FeeDistributorInput"
 
-export async function deployFeeDistributor(params: FeeDistributorIdentityParams) {
-    const factory = getFeeDistributorFactoryContractSigned()
+export async function deployFeeDistributor(fd: FeeDistributorInput) {
+    const params = fd.identityParams
+    if (!params) {
+      throw new Error('No identityParams')
+    }
+
+    const {factory, referenceFd} = await getMatchingFactory(fd)
 
     const tx: ContractTransaction = await factory.createFeeDistributor(
-        params.referenceFeeDistributor,
+        referenceFd,
         params.clientConfig,
         params.referrerConfig, {
             gasLimit: 300000,

scripts/getFdAddressesWithPeriodsFromApi.ts

@@ -0,0 +1,18 @@
+import axios from "axios";
+import {logger} from "./helpers/logger";
+import {FdWithPeriodFromApi} from "./models/FdWithPeriodFromApi";
+
+export async function getFdAddressesWithPeriodsFromApi() : Promise<Record<string, FdWithPeriodFromApi[]>> {
+    logger.info('getFdAddressesWithPeriodsFromApi started')
+
+    if (!process.env.DISTRIBUTORS_URL) {
+        throw new Error("No DISTRIBUTORS_URL in ENV")
+    }
+
+    const result = await axios.get(process.env.DISTRIBUTORS_URL!)
+
+    const array = result.data as Record<string, FdWithPeriodFromApi[]>
+
+    logger.info('getFdAddressesWithPeriodsFromApi finished')
+    return array
+}

scripts/getFeeDistributorInputs.ts

@@ -1,8 +1,10 @@
 import {logger} from "./helpers/logger";
 import {FeeDistributorInput} from "./models/FeeDistributorInput";
-import {getSessionsFromApi} from "./getSessionsFromApi";
-import {DepositManagerApiSession} from "./models/DepositManagerApiSession";
-import {getPubkeysForSessionFromApi} from "./getPubkeysForSessionFromApi";
+import {getFdAddressesWithPeriodsFromApi} from "./getFdAddressesWithPeriodsFromApi";
+import { ethers } from "ethers"
+import { Period } from "./models/Period"
+import { FdWithPeriodFromApi } from "./models/FdWithPeriodFromApi"
+import { getProposers } from "./getProposers"
 
 export async function getFeeDistributorInputs() {
     logger.info('getFeeDistributorInputs started')
@@ -11,109 +13,79 @@ export async function getFeeDistributorInputs() {
         throw new Error("No REFERENCE_FEE_DISTRIBUTOR in ENV")
     }
 
-    const sessionsFromApi = await getSessionsFromApi()
-    logger.info(sessionsFromApi.length + ' sessions from API')
+    const withManual = await getFdAddressesWithPeriodsFromApi()
+    logger.info(Object.keys(withManual).length + ' fd addresses with periods from API with manual')
 
-    const sessionsWithoutInvoices = sessionsFromApi.filter(d => d.address !== d.client_fee_recipient)
-    logger.info(sessionsWithoutInvoices.length + ' sessions without invoices')
-
-    const sessionsGroupedByClient = sessionsWithoutInvoices.reduce((acc: {[key: string]: DepositManagerApiSession[]}, item: DepositManagerApiSession) => {
-        const key = item.client_fee_recipient
-        if (!acc[key]) {
-            acc[key] = []
-        }
-        acc[key].push(item)
-        return acc
-    }, {})
+    const fdAddressesWithPeriodsFromApi = await filterManualSetup(withManual)
+    const fsAddresses = Object.keys(fdAddressesWithPeriodsFromApi)
+    logger.info(fsAddresses.length + ' fd addresses with periods from API')
 
     const now = new Date()
-    const feeDistributorInputs: FeeDistributorInput[] = []
 
-    for (const clientAddress of Object.keys(sessionsGroupedByClient)) {
-        const clientSessions = sessionsGroupedByClient[clientAddress]
+    const feeDistributorInputs: FeeDistributorInput[] = fsAddresses.map(fdAddress => {
+        const periodsFromApi = fdAddressesWithPeriodsFromApi[fdAddress]
 
-        clientSessions.sort((a, b) =>
-            new Date(a.activated_at).getTime() - new Date(b.activated_at).getTime()
-        );
+        const first = periodsFromApi[0]
 
-        for (const session of clientSessions) {
-            session.pubkeys = new Set(await getPubkeysForSessionFromApi(session.session_id))
+        const clientConfig = {
+            recipient: first.client_fee_recipient,
+            basisPoints: first.client_basis_points
         }
 
-        const fdsForPeriodForClient = squashPeriods(clientSessions)
-        fdsForPeriodForClient.sort((a, b) => new Date(a.activated_at).getTime() - new Date(b.activated_at).getTime())
-
-        const feeDistributorInputsForClient: FeeDistributorInput[] = fdsForPeriodForClient.map(d => {
-
-            const referrerConfig = {
-                recipient: d.referrer_fee_recipient,
-                // @ts-ignore
-                basisPoints: d['referrer_basis_points'] as number || 0
-            }
+        const referrerConfig = {
+            recipient: first.referrer_fee_recipient,
+            basisPoints: first['referrer_basis_points'] as number || 0
+        }
 
-            logger.info(d.pubkeys.size + ' pubkeys for ' + d.client_fee_recipient)
+        const periods: Period[] = periodsFromApi.map(pa => ({
+            startDate: new Date(pa.activated_at),
 
-            let endDate = now
+            endDate: pa.deactivated_at
+              ? new Date(pa.deactivated_at)
+              : now,
 
-            const sameClientFds = fdsForPeriodForClient.filter(
-                nd => nd.id !== d.id
-            )
+            pubkeys: pa.validators
+        })).filter(p => p.startDate < p.endDate)
 
-            if (sameClientFds.length) {
-                const laterFd = sameClientFds.find(
-                    fd => new Date(fd.activated_at) > new Date(d.activated_at)
-                )
+        periods.sort((a, b) => a.startDate.getTime() - b.startDate.getTime())
 
-                if (laterFd) {
-                    endDate = new Date(laterFd.activated_at)
-                }
-            }
+        return {
+            fdAddress: ethers.utils.getAddress(fdAddress),
 
-            return {
-                fdAddress: d.address,
+            identityParams: {
+                referenceFeeDistributor: process.env.REFERENCE_FEE_DISTRIBUTOR!,
+                clientConfig,
+                referrerConfig
+            },
 
-                identityParams: {
-                    referenceFeeDistributor: process.env.REFERENCE_FEE_DISTRIBUTOR!,
-                    clientConfig: {recipient: d.client_fee_recipient, basisPoints: d.client_basis_points},
-                    referrerConfig
-                },
+            periods
+        }
+    })
 
-                pubkeys: d.pubkeys,
+    logger.info('getFeeDistributorInputs finished')
+    return feeDistributorInputs
+}
 
-                startDate: new Date(d.activated_at) <= new Date("2024-01-31T20:20:00.000Z")
-                    ? new Date("2024-01-31T20:20:00.000Z")
-                    : new Date(d.activated_at),
+async function filterManualSetup(fds: Record<string, FdWithPeriodFromApi[]>): Promise<Record<string, FdWithPeriodFromApi[]>> {
+    const proposers = await getProposers()
+    const filteredFds = {};
 
-                endDate: endDate <= new Date("2024-01-31T20:20:00.000Z")
-                    ? new Date("2024-01-31T20:20:00.000Z")
-                    : endDate
-            }
-        })
+    Object.entries(fds).forEach(([ethAddress, detailsArray]) => {
+        const filteredDetails = detailsArray.filter(details => {
+            const validators = details.validators;
 
-        feeDistributorInputs.push(...feeDistributorInputsForClient)
-    }
+            const matchingProposers = Object.entries(proposers)
+              .filter(([, proposerDetails]) => proposerDetails.fee_recipient === ethAddress);
 
-    logger.info('getFeeDistributorInputs finished')
-    return feeDistributorInputs
-}
+            return matchingProposers.every(([pubkey]) => validators.includes(pubkey));
+        });
 
-function squashPeriods(withoutInvoices: DepositManagerApiSession[]) {
-    const result: DepositManagerApiSession[] = [];
-    let lastUniqueCombo = null;
-
-    for (const item of withoutInvoices) {
-        const currentCombo = `${item.address}|${item.client_basis_points}`;
-
-        // If the current combination is different from the last unique combination
-        if (currentCombo !== lastUniqueCombo) {
-            result.push(item); // Include this item
-            lastUniqueCombo = currentCombo; // Update last unique combination
-        } else {
-            item.pubkeys.forEach(pk => {
-                result[result.length - 1].pubkeys.add(pk)
-            })
+        if (filteredDetails.length > 0) {
+            // @ts-ignore
+            filteredFds[ethAddress] = filteredDetails;
         }
-    }
+    });
 
-    return result;
+    return filteredFds
 }
+