fix multichain-testing flake (#10915)

Refs: #9934

## Description
Patches over the sequence mismatch in **Override Chain Registry** by retrying.

Doesn't yet solve the wallet provisioning error but I think the above is worth landing. This does add logging to help diagnose the wallet one.

I'll clean up commits before merge. 

### Security Considerations
none

### Scaling Considerations
none

### Documentation Considerations
none

### Testing Considerations
none

### Upgrade Considerations
none

multichain-testing/package.json

@@ -34,7 +34,7 @@
     "ava": "^6.2.0",
     "eslint": "^9.0.0",
     "eslint-config-prettier": "^9.1.0",
-    "execa": "9.1.0",
+    "execa": "^9.5.2",
     "fs-extra": "^11.2.0",
     "patch-package": "^8.0.0",
     "starshipjs": "2.4.1",

multichain-testing/scripts/deploy-cli.ts

@@ -30,7 +30,12 @@ async function main() {
   try {
     const agdTools = await makeAgdTools(console.log, childProcess);
     const deployBuilder = makeDeployBuilder(agdTools, fse.readJSON, execa);
-    await deployBuilder(builder, builderOpts);
+    // XXX this has been flaky so try a second time
+    // see https://github.com/Agoric/agoric-sdk/issues/9934
+    await deployBuilder(builder, builderOpts).catch(err => {
+      console.error('deploy failed, trying again', err);
+      return deployBuilder(builder, builderOpts);
+    });
   } catch (err) {
     console.error(err);
     process.exit(1);

multichain-testing/tools/agd-lib.js

@@ -1,6 +1,9 @@
 // @ts-check
+import { makeTracer } from '@agoric/internal';
 import assert from 'node:assert';
 
+const trace = makeTracer('Agd');
+
 const { freeze } = Object;
 
 const kubectlBinary = 'kubectl';
@@ -136,13 +139,16 @@ export const makeAgd = ({ execFileSync }) => {
           ...(yes ? ['--yes'] : []),
           ...outJson,
         ];
-        console.log('$$$ agd', ...args);
+        trace('$ agd', ...args);
         const out = exec(args, { stdio: ['ignore', 'pipe', 'ignore'] });
         try {
           // XXX approximate type
           /** @type {{ height: string, txhash: string, code: number, codespace: string, raw_log: string }} */
           const detail = JSON.parse(out);
+          trace('agd returned;', detail);
           if (detail.code !== 0) {
+            // FIXME we're getting: account sequence mismatch, expected 30, got 29: incorrect account sequence
+            // Does that mean `broadcast-mode: block` didn't work?
             throw Error(detail.raw_log);
           }
           return detail;

multichain-testing/tools/deploy.ts

@@ -3,9 +3,12 @@ import { createRequire } from 'module';
 import type { AgdTools } from './agd-tools.js';
 import type { CoreEvalPlan } from '@agoric/deploy-script-support/src/writeCoreEvalParts.js';
 import { flags } from './agd-lib.js';
+import { makeTracer } from '@agoric/internal';
 
 const nodeRequire = createRequire(import.meta.url);
 
+const trace = makeTracer('Depl');
+
 export const makeDeployBuilder = (
   tools: AgdTools,
   readJSON: typeof import('fs-extra').readJSON,
@@ -15,7 +18,7 @@ export const makeDeployBuilder = (
     builder: string,
     builderOpts?: Record<string, string | string[]>,
   ) {
-    console.log(`building plan: ${builder}`);
+    trace(`building plan: ${builder}`);
     const args = ['run', builder];
     if (builderOpts) {
       args.push(...flags(builderOpts));
@@ -26,26 +29,28 @@ export const makeDeployBuilder = (
       throw Error('no permit found');
     }
     const plan = await readJSON(`./${match.groups.name}-plan.json`);
-    console.log(plan);
+    trace(plan);
 
-    console.log('copying files to container');
+    trace('copying files to container');
     tools.copyFiles([
       nodeRequire.resolve(`../${plan.script}`),
       nodeRequire.resolve(`../${plan.permit}`),
       ...plan.bundles.map((b: CoreEvalPlan['bundles'][0]) => b.fileName),
     ]);
 
-    console.log('installing bundles');
+    trace('installing bundles');
     await tools.installBundles(
       plan.bundles.map(
         (b: CoreEvalPlan['bundles'][0]) => `/tmp/contracts/${b.bundleID}.json`,
       ),
-      console.log,
+      trace,
     );
 
-    console.log('executing proposal');
+    trace('executing proposal');
     await tools.runCoreEval({
       name: plan.name,
       description: `${plan.name} proposal`,
     });
+
+    trace('done');
   };

multichain-testing/tools/e2e-tools.js

@@ -109,10 +109,11 @@ const installBundle = async (fullPath, opts) => {
   // };
   // const updates = follow('bundles', { delay: explainDelay });
   // await updates.next();
-  const tx = await agd.tx(
-    ['swingset', 'install-bundle', `@${fullPath}`, '--gas', 'auto'],
-    { from, chainId, yes: true },
-  );
+  const tx = await agd.tx(['swingset', 'install-bundle', `@${fullPath}`], {
+    from,
+    chainId,
+    yes: true,
+  });
   assert(tx);
 
   progress({ id, installTx: tx.txhash, height: tx.height });
@@ -157,6 +158,7 @@ export const provisionSmartWallet = async (
     retryUntilCondition,
   },
 ) => {
+  trace('provisionSmartWallet', address);
   // TODO: skip this query if balances is {}
   const vbankEntries = await q.queryData('published.agoricNames.vbankAsset');
   const byName = Object.fromEntries(
@@ -173,6 +175,7 @@ export const provisionSmartWallet = async (
    * @param {bigint} value
    */
   const sendFromWhale = async (denom, value) => {
+    trace('sendFromWhale', address, denom, value);
     const amount = `${value}${denom}`;
     progress({ amount, to: address });
     // TODO: refactor agd.tx to support a per-sender object
@@ -235,7 +238,7 @@ export const provisionSmartWallet = async (
     /** @type {AsyncGenerator<UpdateRecord, void, void>} */
     const updates = q.follow(`published.wallet.${address}`, { delay });
     const txInfo = await sendAction({ method: 'executeOffer', offer });
-    console.debug('spendAction', txInfo);
+    trace('spendAction', txInfo);
     for await (const update of updates) {
       trace('update', address, update);
       if (update.updated !== 'offerStatus' || update.status.id !== offer.id) {
@@ -481,7 +484,7 @@ export const makeE2ETools = async (
     if (typeof info === 'object' && Object.keys(info).length > 0) {
       // XXX normally we have the caller pass in the log function
       // later, but the way blockTool is factored, we have to supply it early.
-      trace({ ...info, delay: ms / 1000 }, '...');
+      trace('delay', { ...info, delay: ms / 1000 }, '...');
     }
     return delay(ms);
   };
@@ -620,7 +623,7 @@ export const seatLike = updates => {
         if ('result' in update.status) sync.result.resolve(result);
         if ('payouts' in update.status && payouts) {
           sync.payouts.resolve(payouts);
-          console.debug('paid out', update.status.id);
+          trace('paid out', update.status.id);
           return;
         }
       }

multichain-testing/tools/faucet-tools.ts

@@ -4,6 +4,9 @@ import { makeFundAndTransfer } from './ibc-transfer.js';
 import type { MultichainRegistry } from './registry.js';
 import type { RetryUntilCondition } from './sleep.js';
 import type { AgdTools } from './agd-tools.js';
+import { makeTracer } from '@agoric/internal';
+
+const trace = makeTracer('Faucet');
 
 type ChainName = string;
 
@@ -29,7 +32,7 @@ export const makeFaucetTools = (
      */
     fundFaucet: async (assets: [ChainName, Denom][], qty = DEFAULT_QTY) => {
       const faucetAddr = agd.keys.showAddress('faucet');
-      console.log(`Faucet address: ${faucetAddr}`);
+      trace(`Faucet address: ${faucetAddr}`);
 
       for (const [chainName, denom] of assets) {
         await fundAndTransfer(chainName, faucetAddr, qty, denom);

multichain-testing/tools/noble-tools.ts

@@ -26,7 +26,8 @@ export const makeNobleTools = (
   }: {
     execFileSync: ExecSync;
   },
-  log: (...args: unknown[]) => void = console.log,
+  log: (...args: unknown[]) => void = (...args) =>
+    console.log('NobleTools', ...args),
 ) => {
   const exec = (
     args: string[],

multichain-testing/yarn.lock

@@ -2939,23 +2939,23 @@ __metadata:
   languageName: node
   linkType: hard
 
-"execa@npm:9.1.0":
-  version: 9.1.0
-  resolution: "execa@npm:9.1.0"
+"execa@npm:^9.5.2":
+  version: 9.5.2
+  resolution: "execa@npm:9.5.2"
   dependencies:
     "@sindresorhus/merge-streams": "npm:^4.0.0"
     cross-spawn: "npm:^7.0.3"
     figures: "npm:^6.1.0"
     get-stream: "npm:^9.0.0"
-    human-signals: "npm:^7.0.0"
+    human-signals: "npm:^8.0.0"
     is-plain-obj: "npm:^4.1.0"
     is-stream: "npm:^4.0.1"
-    npm-run-path: "npm:^5.2.0"
+    npm-run-path: "npm:^6.0.0"
     pretty-ms: "npm:^9.0.0"
     signal-exit: "npm:^4.1.0"
     strip-final-newline: "npm:^4.0.0"
     yoctocolors: "npm:^2.0.0"
-  checksum: 10c0/9a4810b93d098eb0bed48793b61c3aa3e5804867c2c5808cd2a597a6e71738151a74dc792909085ce1d38e89f4b0e078d93ffd165aaca2d9a6728f3616f8e5c0
+  checksum: 10c0/94782a6282e03253224406c29068d18f9095cc251a45d1f19ac3d8f2a9db2cbe32fb8ceb039db1451d8fce3531135a6c0c559f76d634f85416268fc4a6995365
   languageName: node
   linkType: hard
 
@@ -3493,10 +3493,10 @@ __metadata:
   languageName: node
   linkType: hard
 
-"human-signals@npm:^7.0.0":
-  version: 7.0.0
-  resolution: "human-signals@npm:7.0.0"
-  checksum: 10c0/ce0c6d62d2e9bfe529d48f7c7fdf4b8c70fce950eef7850719b4e3f5bc71795ae7d61a3699ce13262bed7847705822601cc81f1921ea6a2906852e16228a94ab
+"human-signals@npm:^8.0.0":
+  version: 8.0.0
+  resolution: "human-signals@npm:8.0.0"
+  checksum: 10c0/e4dac4f7d3eb791ed04129fc6a85bd454a9102d3e3b76c911d0db7057ebd60b2956b435b5b5712aec18960488ede3c21ef7c56e42cdd70760c0d84d3c05cd92e
   languageName: node
   linkType: hard
 
@@ -4381,12 +4381,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"npm-run-path@npm:^5.2.0":
-  version: 5.3.0
-  resolution: "npm-run-path@npm:5.3.0"
+"npm-run-path@npm:^6.0.0":
+  version: 6.0.0
+  resolution: "npm-run-path@npm:6.0.0"
   dependencies:
     path-key: "npm:^4.0.0"
-  checksum: 10c0/124df74820c40c2eb9a8612a254ea1d557ddfab1581c3e751f825e3e366d9f00b0d76a3c94ecd8398e7f3eee193018622677e95816e8491f0797b21e30b2deba
+    unicorn-magic: "npm:^0.3.0"
+  checksum: 10c0/b223c8a0dcd608abf95363ea5c3c0ccc3cd877daf0102eaf1b0f2390d6858d8337fbb7c443af2403b067a7d2c116d10691ecd22ab3c5273c44da1ff8d07753bd
   languageName: node
   linkType: hard
 
@@ -4919,7 +4920,7 @@ __metadata:
     ava: "npm:^6.2.0"
     eslint: "npm:^9.0.0"
     eslint-config-prettier: "npm:^9.1.0"
-    execa: "npm:9.1.0"
+    execa: "npm:^9.5.2"
     fs-extra: "npm:^11.2.0"
     patch-package: "npm:^8.0.0"
     starshipjs: "npm:2.4.1"
@@ -5491,6 +5492,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"unicorn-magic@npm:^0.3.0":
+  version: 0.3.0
+  resolution: "unicorn-magic@npm:0.3.0"
+  checksum: 10c0/0a32a997d6c15f1c2a077a15b1c4ca6f268d574cf5b8975e778bb98e6f8db4ef4e86dfcae4e158cd4c7e38fb4dd383b93b13eefddc7f178dea13d3ac8a603271
+  languageName: node
+  linkType: hard
+
 "unique-filename@npm:^4.0.0":
   version: 4.0.0
   resolution: "unique-filename@npm:4.0.0"