Add scripts to backfill missing vote markers (#776)

* WIP

* Add dump vote marker script

* Add scripts to backfill missing vote markers

* Rm mudlands

packages/account-postgres-sink-service/src/types.ts

@@ -22,6 +22,7 @@ export interface IAccountConfig {
   batchSize: number;
   plugins?: IPluginConfig[];
   ix_side_effects?: IIxSideEffect[];
+  ignore_deletes?: boolean;
 }
 
 export interface IConfig {

packages/account-postgres-sink-service/src/utils/handleAccountWebhook.ts

@@ -90,10 +90,13 @@ export const handleAccountWebhook = async ({
       );
 
       if (isDelete) {
-        await model.destroy({
-          where: { address: account.pubkey },
-          transaction: t,
-        });
+        let ignoreDelete = accounts.find(acc => acc.type == accName)?.ignore_deletes;
+        if (!ignoreDelete) {
+          await model.destroy({
+            where: { address: account.pubkey },
+            transaction: t,
+          });
+        }
       } else {
         sanitized = {
           refreshed_at: new Date().toISOString(),

packages/account-postgres-sink-service/src/utils/upsertProgramAccounts.ts

@@ -198,13 +198,15 @@ export const upsertProgramAccounts = async ({
         }
       );
 
-      await model.destroy({
-        where: {
-          refreshed_at: {
-            [Op.lt]: now,
+      if (!rest.ignore_deletes) {
+        await model.destroy({
+          where: {
+            refreshed_at: {
+              [Op.lt]: now,
+            },
           },
-        },
-      });
+        });
+      }
     } catch (err) {
       console.error(`Error processing account type ${type}:`, err);
     }

packages/helium-admin-cli/package.json

@@ -52,6 +52,7 @@
     "@helium/nft-proxy-sdk": "^0.0.15",
     "@helium/organization-sdk": "^0.0.13",
     "@helium/price-oracle-sdk": "^0.9.19",
+    "@helium/proposal-sdk": "^0.0.15",
     "@helium/spl-utils": "^0.9.19",
     "@helium/treasury-management-sdk": "^0.9.19",
     "@solana/spl-account-compression": "^0.1.7",

packages/helium-admin-cli/src/backfill-vote-markers.ts

@@ -0,0 +1,178 @@
+import AWS from "aws-sdk";
+import fs from "fs";
+import * as pg from "pg";
+import {
+  ARRAY,
+  BOOLEAN,
+  DECIMAL,
+  INTEGER,
+  Model,
+  STRING,
+  Sequelize,
+} from "sequelize";
+import yargs from "yargs/yargs";
+
+export async function run(args: any = process.argv) {
+  const yarg = yargs(args).options({
+    outputPath: {
+      type: "string",
+      describe: "The path to the output file",
+      default: "vote-markers.json",
+    },
+    pgUser: {
+      default: "postgres",
+    },
+    pgPassword: {
+      type: "string",
+    },
+    pgDatabase: {
+      type: "string",
+    },
+    pgHost: {
+      default: "localhost",
+    },
+    pgPort: {
+      default: "5432",
+    },
+    awsRegion: {
+      default: "us-east-1",
+    },
+    noSsl: {
+      type: "boolean",
+      default: false,
+    },
+    voteMarkerJson: {
+      type: "string",
+      describe: "The path to the vote marker json file",
+      required: true,
+    },
+  });
+  const argv = await yarg.argv;
+
+  const voteMarkers = JSON.parse(fs.readFileSync(argv.voteMarkerJson, "utf8"));
+
+  const host = argv.pgHost;
+  const port = Number(argv.pgPort);
+  const database = new Sequelize({
+    host,
+    dialect: "postgres",
+    port,
+    logging: false,
+    dialectModule: pg,
+    username: argv.pgUser,
+    database: argv.pgDatabase,
+    pool: {
+      max: 10,
+      min: 5,
+      acquire: 60000,
+      idle: 10000,
+      validate: (client: any) => {
+        try {
+          client.query("SELECT 1");
+          return true;
+        } catch (err) {
+          return false;
+        }
+      },
+    },
+    hooks: {
+      beforeConnect: async (config: any) => {
+        const isRds = host.includes("rds.amazonaws.com");
+
+        let password = argv.pgPassword;
+        if (isRds && !password) {
+          const signer = new AWS.RDS.Signer({
+            region: process.env.AWS_REGION,
+            hostname: process.env.PGHOST,
+            port,
+            username: process.env.PGUSER,
+          });
+          password = await new Promise((resolve, reject) =>
+            signer.getAuthToken({}, (err, token) => {
+              if (err) {
+                return reject(err);
+              }
+              resolve(token);
+            })
+          );
+          config.dialectOptions = {
+            ssl: {
+              require: false,
+              rejectUnauthorized: false,
+            },
+          };
+        }
+        config.password = password;
+      },
+    },
+  });
+
+  class VoteMarker extends Model {
+    declare address: string;
+    declare pubkey: string;
+    declare voter: string;
+    declare registrar: string;
+    declare proposal: string;
+    declare mint: string;
+    declare choices: number[];
+    declare weight: string;
+    declare bumpSeed: number;
+    declare deprecatedRelinquished: boolean;
+    declare proxyIndex: number;
+    declare rentRefund: string;
+  }
+
+  VoteMarker.init(
+    {
+      address: {
+        type: STRING,
+        primaryKey: true,
+      },
+      voter: {
+        type: STRING,
+        primaryKey: true,
+      },
+      registrar: {
+        type: STRING,
+        primaryKey: true,
+      },
+      proposal: {
+        type: STRING,
+        primaryKey: true,
+      },
+      mint: {
+        type: STRING,
+        primaryKey: true,
+      },
+      choices: {
+        type: ARRAY(INTEGER),
+      },
+      weight: {
+        type: DECIMAL.UNSIGNED,
+      },
+      bumpSeed: {
+        type: INTEGER,
+      },
+      deprecatedRelinquished: {
+        type: BOOLEAN,
+      },
+      proxyIndex: {
+        type: INTEGER,
+      },
+      rentRefund: {
+        type: STRING,
+      },
+    },
+    {
+      sequelize: database,
+      modelName: "vote_marker",
+      tableName: "vote_markers",
+      underscored: true,
+      updatedAt: false,
+    }
+  );
+
+  await VoteMarker.bulkCreate(voteMarkers, {
+    ignoreDuplicates: true, // ON CONFLICT DO NOTHING
+  });
+}