From c70a6705341239eaa3c2bd687684f26829e2e2a0 Mon Sep 17 00:00:00 2001 From: Noah Prince <83885631+ChewingGlass@users.noreply.github.com> Date: Fri, 24 Jan 2025 16:43:19 -0600 Subject: [PATCH] Add scripts to backfill missing vote markers (#776) * WIP * Add dump vote marker script * Add scripts to backfill missing vote markers * Rm mudlands --- .../src/types.ts | 1 + .../src/utils/handleAccountWebhook.ts | 11 +- .../src/utils/upsertProgramAccounts.ts | 14 +- packages/helium-admin-cli/package.json | 1 + .../src/backfill-vote-markers.ts | 178 ++++++++++ .../helium-admin-cli/src/dump-vote-markers.ts | 333 ++++++++++++++++++ packages/helium-admin-cli/yarn.deploy.lock | 22 ++ yarn.lock | 1 + 8 files changed, 551 insertions(+), 10 deletions(-) create mode 100644 packages/helium-admin-cli/src/backfill-vote-markers.ts create mode 100644 packages/helium-admin-cli/src/dump-vote-markers.ts diff --git a/packages/account-postgres-sink-service/src/types.ts b/packages/account-postgres-sink-service/src/types.ts index cad0af788..dbb463f8b 100644 --- a/packages/account-postgres-sink-service/src/types.ts +++ b/packages/account-postgres-sink-service/src/types.ts @@ -22,6 +22,7 @@ export interface IAccountConfig { batchSize: number; plugins?: IPluginConfig[]; ix_side_effects?: IIxSideEffect[]; + ignore_deletes?: boolean; } export interface IConfig { diff --git a/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts b/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts index 707d46af6..7529473ba 100644 --- a/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts +++ b/packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts @@ -90,10 +90,13 @@ export const handleAccountWebhook = async ({ ); if (isDelete) { - await model.destroy({ - where: { address: account.pubkey }, - transaction: t, - }); + let ignoreDelete = accounts.find(acc => acc.type == accName)?.ignore_deletes; + if (!ignoreDelete) { + await model.destroy({ + where: { address: account.pubkey }, + transaction: t, + }); + } } else { sanitized = { refreshed_at: new Date().toISOString(), diff --git a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts index d7e37e138..99d037118 100644 --- a/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts +++ b/packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts @@ -198,13 +198,15 @@ export const upsertProgramAccounts = async ({ } ); - await model.destroy({ - where: { - refreshed_at: { - [Op.lt]: now, + if (!rest.ignore_deletes) { + await model.destroy({ + where: { + refreshed_at: { + [Op.lt]: now, + }, }, - }, - }); + }); + } } catch (err) { console.error(`Error processing account type ${type}:`, err); } diff --git a/packages/helium-admin-cli/package.json b/packages/helium-admin-cli/package.json index 7c5012cf5..f382dd8ba 100644 --- a/packages/helium-admin-cli/package.json +++ b/packages/helium-admin-cli/package.json @@ -52,6 +52,7 @@ "@helium/nft-proxy-sdk": "^0.0.15", "@helium/organization-sdk": "^0.0.13", "@helium/price-oracle-sdk": "^0.9.19", + "@helium/proposal-sdk": "^0.0.15", "@helium/spl-utils": "^0.9.19", "@helium/treasury-management-sdk": "^0.9.19", "@solana/spl-account-compression": "^0.1.7", diff --git a/packages/helium-admin-cli/src/backfill-vote-markers.ts b/packages/helium-admin-cli/src/backfill-vote-markers.ts new file mode 100644 index 000000000..47b0bd9ef --- /dev/null +++ b/packages/helium-admin-cli/src/backfill-vote-markers.ts @@ -0,0 +1,178 @@ +import AWS from "aws-sdk"; +import fs from "fs"; +import * as pg from "pg"; +import { + ARRAY, + BOOLEAN, + DECIMAL, + INTEGER, + Model, + STRING, + Sequelize, +} from "sequelize"; +import yargs from "yargs/yargs"; + +export async function run(args: any = process.argv) { + const yarg = yargs(args).options({ + outputPath: { + type: "string", + describe: "The path to the output file", + default: "vote-markers.json", + }, + pgUser: { + default: "postgres", + }, + pgPassword: { + type: "string", + }, + pgDatabase: { + type: "string", + }, + pgHost: { + default: "localhost", + }, + pgPort: { + default: "5432", + }, + awsRegion: { + default: "us-east-1", + }, + noSsl: { + type: "boolean", + default: false, + }, + voteMarkerJson: { + type: "string", + describe: "The path to the vote marker json file", + required: true, + }, + }); + const argv = await yarg.argv; + + const voteMarkers = JSON.parse(fs.readFileSync(argv.voteMarkerJson, "utf8")); + + const host = argv.pgHost; + const port = Number(argv.pgPort); + const database = new Sequelize({ + host, + dialect: "postgres", + port, + logging: false, + dialectModule: pg, + username: argv.pgUser, + database: argv.pgDatabase, + pool: { + max: 10, + min: 5, + acquire: 60000, + idle: 10000, + validate: (client: any) => { + try { + client.query("SELECT 1"); + return true; + } catch (err) { + return false; + } + }, + }, + hooks: { + beforeConnect: async (config: any) => { + const isRds = host.includes("rds.amazonaws.com"); + + let password = argv.pgPassword; + if (isRds && !password) { + const signer = new AWS.RDS.Signer({ + region: process.env.AWS_REGION, + hostname: process.env.PGHOST, + port, + username: process.env.PGUSER, + }); + password = await new Promise((resolve, reject) => + signer.getAuthToken({}, (err, token) => { + if (err) { + return reject(err); + } + resolve(token); + }) + ); + config.dialectOptions = { + ssl: { + require: false, + rejectUnauthorized: false, + }, + }; + } + config.password = password; + }, + }, + }); + + class VoteMarker extends Model { + declare address: string; + declare pubkey: string; + declare voter: string; + declare registrar: string; + declare proposal: string; + declare mint: string; + declare choices: number[]; + declare weight: string; + declare bumpSeed: number; + declare deprecatedRelinquished: boolean; + declare proxyIndex: number; + declare rentRefund: string; + } + + VoteMarker.init( + { + address: { + type: STRING, + primaryKey: true, + }, + voter: { + type: STRING, + primaryKey: true, + }, + registrar: { + type: STRING, + primaryKey: true, + }, + proposal: { + type: STRING, + primaryKey: true, + }, + mint: { + type: STRING, + primaryKey: true, + }, + choices: { + type: ARRAY(INTEGER), + }, + weight: { + type: DECIMAL.UNSIGNED, + }, + bumpSeed: { + type: INTEGER, + }, + deprecatedRelinquished: { + type: BOOLEAN, + }, + proxyIndex: { + type: INTEGER, + }, + rentRefund: { + type: STRING, + }, + }, + { + sequelize: database, + modelName: "vote_marker", + tableName: "vote_markers", + underscored: true, + updatedAt: false, + } + ); + + await VoteMarker.bulkCreate(voteMarkers, { + ignoreDuplicates: true, // ON CONFLICT DO NOTHING + }); +} diff --git a/packages/helium-admin-cli/src/dump-vote-markers.ts b/packages/helium-admin-cli/src/dump-vote-markers.ts new file mode 100644 index 000000000..08c0d7509 --- /dev/null +++ b/packages/helium-admin-cli/src/dump-vote-markers.ts @@ -0,0 +1,333 @@ +import * as anchor from "@coral-xyz/anchor"; +import { VoterStakeRegistry } from "@helium/idls/lib/types/voter_stake_registry"; +import { organizationKey } from "@helium/organization-sdk"; +import { init as initProposal } from "@helium/proposal-sdk"; +import { init as initVsr } from "@helium/voter-stake-registry-sdk"; +import { PublicKey } from "@solana/web3.js"; +import b58 from "bs58"; +import os from "os"; +import yargs from "yargs/yargs"; +import { + loadKeypair +} from "./utils"; + +type VoteMarkerV0 = anchor.IdlAccounts["voteMarkerV0"] & { address: PublicKey }; + +export async function run(args: any = process.argv) { + const yarg = yargs(args).options({ + wallet: { + alias: "k", + describe: "Anchor wallet keypair", + default: `${os.homedir()}/.config/solana/id.json`, + }, + url: { + alias: "u", + default: "http://127.0.0.1:8899", + describe: "The solana url", + }, + outputPath: { + type: "string", + describe: "The path to the output file", + default: "vote-markers.json", + }, + }); + + const argv = await yarg.argv; + process.env.ANCHOR_WALLET = argv.wallet; + process.env.ANCHOR_PROVIDER_URL = argv.url; + anchor.setProvider(anchor.AnchorProvider.local(argv.url)); + + + const provider = anchor.getProvider() as anchor.AnchorProvider; + const wallet = new anchor.Wallet(loadKeypair(argv.wallet)); + const hvsrProgram = await initVsr(provider); + const proposalProgram = await initProposal(provider); + let myOrgs = new Set([ + organizationKey("Helium")[0], + organizationKey("Helium MOBILE")[0], + organizationKey("Helium IOT")[0], + ].map(k => k.toBase58())); + const proposals = (await proposalProgram.account.proposalV0.all()).filter(p => myOrgs.has(p.account.namespace.toBase58())); + + // Track votes in a map: proposalKey -> voter marker id --> VoteMarkerV0 + const votesByProposal = new Map>(); + for (const proposal of proposals) { + votesByProposal.set(proposal.publicKey.toBase58(), new Map()); + } + + // Add position cache + const positionCache = new Map(); + + for (const proposal of proposals) { + console.log(`Getting vote markers for ${proposal.publicKey.toBase58()}`); + + let signatures: anchor.web3.ConfirmedSignatureInfo[] = []; + let lastSig: string | undefined = undefined; + + // // Keep fetching until we get all signatures + while (true) { + const sigs = await provider.connection.getSignaturesForAddress( + proposal.publicKey, + { before: lastSig, limit: 1000 }, + "confirmed" + ); + + if (sigs.length === 0) break; + + signatures.push(...sigs); + lastSig = sigs[sigs.length - 1].signature; + + // If we got less than 1000, we've hit the end + if (sigs.length < 1000) break; + } + + console.log("signatures", signatures.length); + + const hvsrCoder = new anchor.BorshInstructionCoder(hvsrProgram.idl); + const proposalCoder = new anchor.BorshInstructionCoder(proposalProgram.idl); + + // Process signatures in chunks of 100 + const chunkSize = 100; + signatures = signatures.reverse() + for (let i = 0; i < signatures.length; i += chunkSize) { + const chunk = signatures.slice(i, i + chunkSize); + const txs = await provider.connection.getTransactions( + chunk.map(sig => sig.signature), + { maxSupportedTransactionVersion: 0, commitment: "confirmed" }, + ); + + for (const tx of txs) { + if (!tx?.meta || tx.meta.err) continue; + + let message = tx.transaction.message; + let index = -1; + for (const ix of message.compiledInstructions) { + index++; + try { + // Check if instruction is from VSR program + if (message.staticAccountKeys[ix.programIdIndex].toBase58() !== hvsrProgram.programId.toBase58()) continue; + + let decoded = hvsrCoder.decode(Buffer.from(ix.data)); + + if (!decoded) continue; + + let formatted = hvsrCoder.format(decoded, ix.accountKeyIndexes.map(i => { + return { + pubkey: message.staticAccountKeys[i] || PublicKey.default, + isSigner: false, + isWritable: false, + } + })); + + if (!formatted) continue; + + // Handle vote instruction + if (decoded.name === "voteV0") { + const voter = formatted.accounts.find( + (acc) => acc.name === "Voter" + )?.pubkey; + const registrar = formatted.accounts.find( + (acc) => acc.name === "Registrar" + )?.pubkey; + const mint = formatted.accounts.find( + (acc) => acc.name === "Mint" + )?.pubkey; + const proposal = formatted.accounts.find( + (acc) => acc.name === "Proposal" + )?.pubkey; + const marker = formatted.accounts.find( + (acc) => acc.name === "Marker" + )?.pubkey; + const innerIxs = tx.meta.innerInstructions?.find( + (ix) => ix.index === index + ); + const innerVoteIx = innerIxs?.instructions.find( + (ix) => + message.staticAccountKeys[ix.programIdIndex]?.toBase58() === + proposalProgram.programId.toBase58() + ); + const innerVoteDecoded = proposalCoder.decode( + Buffer.from(b58.decode(innerVoteIx?.data || "")) + ); + if (!innerVoteDecoded) { + console.log("innerVoteDecoded missing", index, tx.meta.innerInstructions, innerVoteIx, innerIxs, chunk); + }; + // @ts-ignore + let { weight, choice } = innerVoteDecoded!.data.args; + let propMap = votesByProposal.get(proposal!.toBase58()); + + let voteMarker = propMap!.get(marker!.toBase58()); + if (!voteMarker) { + voteMarker = { + address: marker, + voter, + registrar, + proposal, + mint, + // @ts-ignore + choices: [choice], + weight: weight, + bumpSeed: 0, + deprecatedRelinquished: false, + proxyIndex: 0, + rentRefund: PublicKey.default, + } as VoteMarkerV0; + } + if (!voteMarker.choices.some((c) => c === choice)) { + voteMarker.choices.push(choice); + } + if (!voteMarker.weight.eq(weight)) { + voteMarker.weight = weight; + } + propMap!.set(marker!.toBase58(), voteMarker); + } else if (decoded.name === "proxiedVoteV0") { + const voter = formatted.accounts.find( + (acc) => acc.name === "Voter" + )?.pubkey; + const position = formatted.accounts.find( + (acc) => acc.name === "Position" + )?.pubkey; + const registrar = formatted.accounts.find( + (acc) => acc.name === "Registrar" + )?.pubkey; + const proposal = formatted.accounts.find( + (acc) => acc.name === "Proposal" + )?.pubkey; + const marker = formatted.accounts.find( + (acc) => acc.name === "Marker" + )?.pubkey; + + // Use cache for position lookup + let mint: PublicKey; + if (position) { + const positionKey = position.toBase58(); + if (!positionCache.has(positionKey)) { + const posData = await hvsrProgram.account.positionV0.fetchNullable(position); + positionCache.set(positionKey, { mint: posData?.mint || PublicKey.default }); + } + mint = positionCache.get(positionKey)!.mint; + } else { + mint = PublicKey.default; + } + + const innerIxs = tx.meta.innerInstructions?.find( + (ix) => ix.index === index + ); + const innerVoteIx = innerIxs?.instructions.find( + (ix) => + message.staticAccountKeys[ix.programIdIndex]?.toBase58() === + proposalProgram.programId.toBase58() + ); + const innerVoteDecoded = proposalCoder.decode( + Buffer.from(b58.decode(innerVoteIx?.data || "")) + ); + if (!innerVoteDecoded) { + console.log( + "innerVoteDecoded missing", + index, + tx.meta.innerInstructions, + innerVoteIx, + innerIxs, + chunk + ); + } + // @ts-ignore + let { weight, choice } = innerVoteDecoded!.data.args; + let propMap = votesByProposal.get(proposal!.toBase58()); + let voteMarker = propMap!.get(marker!.toBase58()); + if (!voteMarker) { + voteMarker = { + address: marker, + voter, + registrar, + proposal, + mint, + // @ts-ignore + choices: [choice], + weight, + bumpSeed: 0, + deprecatedRelinquished: false, + proxyIndex: 0, + rentRefund: PublicKey.default, + } as VoteMarkerV0; + } + if (!voteMarker.choices.some((c) => c === choice)) { + voteMarker.choices.push(choice); + } + if (!voteMarker.weight.eq(weight)) { + voteMarker.weight = weight; + } + propMap!.set(marker!.toBase58(), voteMarker); + } else if (decoded.name === "relinquishVoteV1" || decoded.name === "proxiedRelinquishVoteV0") { + const firstIsSigner = message.isAccountSigner(ix.accountKeyIndexes[0]); + // HACK: At some point we removed rent refund as first account and made it the last account. + if (firstIsSigner && decoded.name === "relinquishVoteV1") { + const len = ix.accountKeyIndexes.length; + let refund = ix.accountKeyIndexes.shift(); + if (len != 12) { // super legacy + ix.accountKeyIndexes.push(refund!); + } + decoded = hvsrCoder.decode(Buffer.from(ix.data))!; + formatted = hvsrCoder.format( + decoded!, + ix.accountKeyIndexes.map((i) => { + return { + pubkey: message.staticAccountKeys[i] || PublicKey.default, + isSigner: false, + isWritable: false, + }; + }) + )!; + } + + const marker = formatted.accounts.find( + (acc) => acc.name === "Marker" + )?.pubkey; + const proposal = formatted.accounts.find( + (acc) => acc.name === "Proposal" + )?.pubkey; + const propMap = votesByProposal.get(proposal!.toBase58()); + let voteMarker = propMap!.get(marker!.toBase58()); + if (voteMarker) { + // @ts-ignore + voteMarker.choices = voteMarker.choices.filter(c => c !== decoded.data.args.choice); + if (voteMarker.choices.length === 0) { + propMap!.delete(marker!.toBase58()); + } else { + propMap!.set(marker!.toBase58(), voteMarker); + } + } + } + } catch (e) { + console.log("error", index, tx.transaction.signatures[0]); + throw e; + // Skip instructions that can't be decoded + continue; + } + } + } + } + } + + let flattened = Array.from(votesByProposal.values()).flatMap((markers) => { + return Array.from(markers.values()).map(marker => ({ + address: marker.address.toBase58(), + voter: marker.voter.toBase58(), + registrar: marker.registrar.toBase58(), + proposal: marker.proposal.toBase58(), + mint: marker.mint.toBase58(), + choices: marker.choices, + weight: marker.weight.toString(), + bumpSeed: marker.bumpSeed, + deprecatedRelinquished: marker.deprecatedRelinquished, + proxyIndex: marker.proxyIndex, + rentRefund: marker.rentRefund.toBase58(), + })) + }); + // Write results to file + const fs = require('fs'); + fs.writeFileSync( + argv.outputPath, + JSON.stringify(flattened, null, 2) + ); +} diff --git a/packages/helium-admin-cli/yarn.deploy.lock b/packages/helium-admin-cli/yarn.deploy.lock index dde386b53..d4cb6c560 100644 --- a/packages/helium-admin-cli/yarn.deploy.lock +++ b/packages/helium-admin-cli/yarn.deploy.lock @@ -331,6 +331,7 @@ __metadata: "@helium/nft-proxy-sdk": ^0.0.15 "@helium/organization-sdk": ^0.0.13 "@helium/price-oracle-sdk": ^0.9.19 + "@helium/proposal-sdk": ^0.0.15 "@helium/spl-utils": ^0.9.19 "@helium/treasury-management-sdk": ^0.9.19 "@solana/spl-account-compression": ^0.1.7 @@ -465,6 +466,16 @@ __metadata: languageName: node linkType: hard +"@helium/modular-governance-idls@npm:^0.0.15": + version: 0.0.15 + resolution: "@helium/modular-governance-idls@npm:0.0.15" + dependencies: + "@coral-xyz/anchor": ^0.28.0 + "@solana/web3.js": ^1.78.4 + checksum: cbfaa2a83b074fb24d138de1901acaa7a0cf0de715bdabb3fdad9ab04b59a34e5f6c02216fdb6b6d78d4fa48bf80fe76d2c657107458f9d9b53d64aa532bc547 + languageName: node + linkType: hard + "@helium/nft-proxy-sdk@npm:^0.0.15": version: 0.0.15 resolution: "@helium/nft-proxy-sdk@npm:0.0.15" @@ -533,6 +544,17 @@ __metadata: languageName: node linkType: hard +"@helium/proposal-sdk@npm:^0.0.15": + version: 0.0.15 + resolution: "@helium/proposal-sdk@npm:0.0.15" + dependencies: + "@coral-xyz/anchor": ^0.28.0 + "@helium/anchor-resolvers": ^0.5.0 + "@helium/modular-governance-idls": ^0.0.15 + checksum: b127e09f428c7f8076341bb692e50bfa19f632dd887502e9d6464fb6066f79975c0fabca223c298c7427c9c6560c49d348f5c33674fd9cc4570f532b34e345dc + languageName: node + linkType: hard + "@helium/rewards-oracle-sdk@^0.9.19": version: 0.0.0-use.local resolution: "@helium/rewards-oracle-sdk@workspace:packages/rewards-oracle-sdk" diff --git a/yarn.lock b/yarn.lock index b3d865918..ceb8b83d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -927,6 +927,7 @@ __metadata: "@helium/nft-proxy-sdk": ^0.0.15 "@helium/organization-sdk": ^0.0.13 "@helium/price-oracle-sdk": ^0.9.19 + "@helium/proposal-sdk": ^0.0.15 "@helium/spl-utils": ^0.9.19 "@helium/treasury-management-sdk": ^0.9.19 "@solana/spl-account-compression": ^0.1.7