From a7ded0eb8945b21a837b48af9d9628cb7f054ed0 Mon Sep 17 00:00:00 2001 From: Stanislav Lysak Date: Wed, 2 Oct 2024 19:54:28 +0300 Subject: [PATCH 1/2] FET-1564: Put test-env into its own repo --- package.json | 1 - packages/ens-test-env/.gitignore | 1 - packages/ens-test-env/README.md | 140 ------ packages/ens-test-env/package.json | 32 -- packages/ens-test-env/src/config.d.ts | 19 - packages/ens-test-env/src/docker-compose.yml | 63 --- .../src/ens-test-env.example.config.js | 28 -- packages/ens-test-env/src/fetch-data.js | 150 ------- packages/ens-test-env/src/index.d.ts | 1 - packages/ens-test-env/src/index.js | 139 ------ packages/ens-test-env/src/manager.js | 407 ------------------ packages/ens-test-env/src/subgraph.js | 40 -- packages/ens-test-env/tsconfig.json | 3 - 13 files changed, 1024 deletions(-) delete mode 100644 packages/ens-test-env/.gitignore delete mode 100644 packages/ens-test-env/README.md delete mode 100644 packages/ens-test-env/package.json delete mode 100644 packages/ens-test-env/src/config.d.ts delete mode 100644 packages/ens-test-env/src/docker-compose.yml delete mode 100644 packages/ens-test-env/src/ens-test-env.example.config.js delete mode 100644 packages/ens-test-env/src/fetch-data.js delete mode 100644 packages/ens-test-env/src/index.d.ts delete mode 100755 packages/ens-test-env/src/index.js delete mode 100644 packages/ens-test-env/src/manager.js delete mode 100644 packages/ens-test-env/src/subgraph.js delete mode 100644 packages/ens-test-env/tsconfig.json diff --git a/package.json b/package.json index 23c342c3..7cb3f33b 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,6 @@ "packages/*" ], "scripts": { - "publish:local:ens-test-env": "yalc publish packages/ens-test-env --push --up", "publish:local:ensjs": "yalc publish packages/ensjs --push --up", "chgset:version": "changeset version && pnpm install", "chgset:run": "changeset", diff --git a/packages/ens-test-env/.gitignore b/packages/ens-test-env/.gitignore deleted file mode 100644 index 7a694c96..00000000 --- a/packages/ens-test-env/.gitignore +++ /dev/null @@ -1 +0,0 @@ -LICENSE \ No newline at end of file diff --git a/packages/ens-test-env/README.md b/packages/ens-test-env/README.md deleted file mode 100644 index 55f383f8..00000000 --- a/packages/ens-test-env/README.md +++ /dev/null @@ -1,140 +0,0 @@ -# ens-test-env - -## How it works - -The testing environment used here is implemented in the stateless and stateful tests for ENS app. -The environment consists of two parts: the ganache Ethereum node, and the docker graph instance. -Which environment type you use is dependent on your testing circumstances. - -## Configuration - -There should be a file named `ens-test-env.config.js` in your project's root directory. -You can add a JSDoc type import to import the config type. - -```js -/** - * @type {import('@ensdomains/ens-test-env').ENSTestEnvConfig} - **/ -module.exports = { - deployCommand: 'yarn hardhat deploy', - tenderly: { - user: 'ens', - project: 'core', - key: 'tenderly-key', - }, - archive: { - subgraphId: 'QmXxAE7Urtv6TPa8o8XmPwLVQNbH6r35hRKHP63udTxTNa', - epochTime: 1646894980, - blockNumber: 12066620, - baseUrl: 'https://storage.googleapis.com/ens-manager-build-data', - network: 'mainnet', - }, - docker: { - file: './docker-compose.yml', - sudo: false, - }, - ethereum: { - chain: { - chainId: 0, - }, - fork: { - url: 'https://example.com', - }, - wallet: { - mnemonic: 'test test test test test test test test test test test junk', - unlockedAccounts: ['0x0000000000000000000000000000000000000000'], - }, - }, - graph: { - bypassLocal: false, - }, - scripts: [ - { - command: 'example', - name: 'example', - prefixColor: 'blue.bold', - cwd: path.resolve('./'), - finishOnExit: true, - waitForGraph: true, - }, - ], - paths: { - archives: './archives', - data: './data', - }, -} -``` - -## Environment Types - -### Stateless - -For most testing situations you can use the default settings, which will create a fresh mainnet -fork from a specified block as well as deploying a fresh subgraph with the same specified block -as it's start block. - -Tests should ideally be designed to be stateless where possible, which entails not using hardcoded -addresses and not relying on any specific blockchain/graph state. This allows for a much higher -test reliability and low maintenance. - -### Stateful - -Some tests may require a specific existing state, for example if a test relies on an old deployment -of a contract which can no longer be accurately replicated from a fresh mainnet fork. The stateful -environment uses pre-existing subgraph data at a specified block to allow full state access prior -to the mainnet fork. - -The stateful environment can also be used to more closely replicate a production environment for -true full end-to-end tests. You may also want to use this environment for testing with your own -personal wallet without using mainnet. - -The downside of using the stateful environment is that a test can potentially become unreliable if -one of it's dependencies changes. Alongside reliability, running a stateful test most of the time -will require access to a specific private key. Given this, you should try to avoid writing stateful -tests wherever possible. - -## Contract deployments - -Contract deployments are a small but necessary part of testing, you can deploy contracts to -both stateless and stateful environments. After the locally tested contract is deployed, the -deployment script should be left in the repo to serve as an archive. - -## Updating the graph-node dataset - -Generally, you will want to set a graft variable in the `subgraph.yaml` file for the subgraph. You -can find more about the ENS subgraph [here](https://github.com/ensdomains/ens-subgraph). You'll also -documentation for grafting available [here](https://thegraph.com/docs/en/developer/create-subgraph-hosted/#grafting-onto-existing-subgraphs). - -To update the graph-node dataset, the BLOCK_HEIGHT variable must be changed within the `.env` file. -It should be set to the same value as the graft block. - -If the dataset is a dependency for a local test, you will need to first let your local graph-node -dataset update so that your test can pass. - -Once your data is up to date, you can run - -```bash -yarn ens-test-env data --compress -``` - -in this directory, which will give you a archive file for your dataset. - -### Dataset naming scheme - -```js -const file = `data_${BLOCK_HEIGHT}_${SUBGRAPH_ID}_${EPOCH_TIME}_${NETWORK}.archive` -// e.g. data_14119046_QmTmU4syjQb8gfNq8TCQGv441qp2zQMNKnQ4smjKhpLQ6F_1643850493_ropsten.archive.tar.lz4 -``` - -## Running the environment - -After this you can run: - -```bash -# Start -yarn ens-test-env start -# Load data only -yarn ens-test-env data --load -# Export generated data -yarn ens-test-env data --compress -``` diff --git a/packages/ens-test-env/package.json b/packages/ens-test-env/package.json deleted file mode 100644 index d9ca2d4c..00000000 --- a/packages/ens-test-env/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "@ensdomains/ens-test-env", - "type": "module", - "version": "0.5.0-beta.1", - "files": [ - "src" - ], - "types": "src/index.d.ts", - "scripts": { - "prepublish": "cp ../../LICENSE ./", - "ver": "pnpm version --no-workspaces-update" - }, - "engines": { - "node": ">=18" - }, - "dependencies": { - "@ethersproject/wallet": "^5.6.0", - "ansi-colors": "^4.1.1", - "cli-progress": "^3.10.0", - "commander": "^9.3.0", - "concurrently": "^7.1.0", - "docker-compose": "^0.24.7", - "dotenv": "^16.0.1", - "js-yaml": "^4.1.0", - "lz4": "^0.6.5", - "progress-stream": "^2.0.0", - "tar-fs": "^2.1.1", - "wait-on": "^6.0.1" - }, - "preferUnplugged": true, - "bin": "./src/index.js" -} diff --git a/packages/ens-test-env/src/config.d.ts b/packages/ens-test-env/src/config.d.ts deleted file mode 100644 index 69bafd9f..00000000 --- a/packages/ens-test-env/src/config.d.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { ConcurrentlyCommandInput } from 'concurrently' - -/** - * ens-test-env configuration object - * @see [configuration documentation](https://github.com/ensdomains/ensjs-v3/tree/main/packages/ens-test-env/) - */ -export interface ENSTestEnvConfig { - deployCommand?: string - buildCommand?: string - labelHashes?: { hash: string; label: string }[] - scripts?: (ConcurrentlyCommandInput & { - finishOnExit?: boolean - })[] - paths?: { - data?: string - archive?: string - composeFile?: string - } -} diff --git a/packages/ens-test-env/src/docker-compose.yml b/packages/ens-test-env/src/docker-compose.yml deleted file mode 100644 index d5fd8b85..00000000 --- a/packages/ens-test-env/src/docker-compose.yml +++ /dev/null @@ -1,63 +0,0 @@ -version: '3' -services: - anvil: - image: ghcr.io/ensdomains/anvil:next - ports: - - '8545:8545' - entrypoint: /bin/sh -c 'anvil --chain-id 1337 --gas-limit 50000000 --timestamp 1640995200 $ANVIL_EXTRA_ARGS' - environment: - ANVIL_IP_ADDR: '0.0.0.0' - graph-node: - image: graphprotocol/graph-node:v0.30.0 - ports: - - '8000:8000' - - '8001:8001' - - '8020:8020' - - '8030:8030' - - '8040:8040' - depends_on: - - ipfs - - postgres - - anvil - environment: - postgres_host: postgres - postgres_user: graph-node - postgres_pass: let-me-in - postgres_db: graph-node - ipfs: 'ipfs:5001' - ethereum: 'mainnet:http://anvil:8545' - GRAPH_LOG: $GRAPH_LOG_LEVEL - GRAPH_ALLOW_NON_DETERMINISTIC_IPFS: 'true' - GRAPH_ETHEREUM_CLEANUP_BLOCKS: 'true' - extra_hosts: - - 'host.docker.internal:host-gateway' - ipfs: - image: ipfs/kubo:v0.16.0 - ports: - - '5001:5001' - volumes: - - $DATA_FOLDER/ipfs:/data/ipfs - postgres: - image: postgres:14-alpine - ports: - - '5432:5432' - command: ['postgres', '-cshared_preload_libraries=pg_stat_statements'] - environment: - POSTGRES_USER: graph-node - POSTGRES_PASSWORD: let-me-in - POSTGRES_DB: graph-node - POSTGRES_INITDB_ARGS: "-E UTF8 --locale=C" - volumes: - - $DATA_FOLDER/postgres:/var/lib/postgresql/data - metadata: - image: ghcr.io/ensdomains/ens-metadata-service:latest - depends_on: - - anvil - - graph-node - ports: - - '8080:8080' - environment: - - ADDRESS_ETH_REGISTRAR=$ADDRESS_ETH_REGISTRAR - - ADDRESS_NAME_WRAPPER=$ADDRESS_NAME_WRAPPER - - RPC_PROVIDER=http://anvil:8545 - - SUBGRAPH_URL=http://graph-node:8000/subgraphs/name/graphprotocol/ens \ No newline at end of file diff --git a/packages/ens-test-env/src/ens-test-env.example.config.js b/packages/ens-test-env/src/ens-test-env.example.config.js deleted file mode 100644 index 5bbe08d9..00000000 --- a/packages/ens-test-env/src/ens-test-env.example.config.js +++ /dev/null @@ -1,28 +0,0 @@ -const path = require('path') - -/** - * @type {import('./config').ENSTestEnvConfig} - * */ -module.exports = { - deployCommand: 'yarn hardhat deploy', - buildCommand: 'yarn build', - labelHashes: [ - { - hash: '0x4f5b812789fc606be1b3b16908db13fc7a9adf7ca72641f84d75b47069d3d7f0', - label: 'eth', - }, - ], - scripts: [ - { - command: 'example', - name: 'example', - prefixColor: 'blue.bold', - cwd: path.resolve('./'), - finishOnExit: true, - }, - ], - paths: { - data: './data', - composeFile: './docker-compose.yml', - }, -} diff --git a/packages/ens-test-env/src/fetch-data.js b/packages/ens-test-env/src/fetch-data.js deleted file mode 100644 index 0d674cd2..00000000 --- a/packages/ens-test-env/src/fetch-data.js +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env node - -/* eslint-disable */ -import colors from 'ansi-colors' -import cliProgress from 'cli-progress' -import 'dotenv/config' -import fs from 'node:fs' -import lz4 from 'lz4' -import progress from 'progress-stream' -import { pipeline } from 'stream' -import tar from 'tar-fs' -import { promisify } from 'node:util' - -const createProgressBar = (name, hasSpeed) => - new cliProgress.SingleBar({ - format: - `${name} Progress |` + - colors.cyan('{bar}') + - '| {percentage}% || {value}/{total} GB' + - (hasSpeed ? ' || Speed: {speed} MB/s' : ''), - barCompleteChar: '\u2588', - barIncompleteChar: '\u2591', - hideCursor: true, - fps: 5, - formatValue: (value, _, type) => { - switch (type) { - case 'value': - return bytesToGb(value).toFixed(2) - case 'total': - return bytesToGb(value).toFixed(2) - default: - return value - } - }, - }) - -const extractProgressBar = createProgressBar('Extract', true) -const compressProgressBar = createProgressBar('Compress', false) - -let localPath, dataPath - -const streamPipeline = promisify(pipeline) -const streamOpts = { - highWaterMark: 67108864, // 64MB -} - -const bytesToGb = (bytes) => bytes * 9.3132257461548e-10 -const bytesToMb = (bytes) => bytes * 9.5367431640625e-7 - -async function compressToArchive() { - return new Promise(async (resolve, reject) => { - let initial = true - const encoder = lz4.createEncoderStream({}) - const output = fs.createWriteStream(localPath + '.tar.lz4', streamOpts) - const archive = tar.pack(dataPath) - const progressStream = progress({}) - - progressStream.on('progress', (progress) => { - if (initial) { - compressProgressBar.start(progress.length, 0, { - speed: 'N/A', - }) - initial = false - } else { - compressProgressBar.update(progress.transferred, { - speed: bytesToMb(progress.speed).toFixed(2), - }) - } - }) - - compressProgressBar.start() - await streamPipeline(archive, encoder, output) - .then(() => { - compressProgressBar.stop() - resolve() - }) - .catch((err) => { - compressProgressBar.stop() - reject(err.message) - }) - }) -} - -async function decompressToOutput() { - return new Promise(async (resolve, reject) => { - if (fs.existsSync(dataPath)) - await fs.promises.rm(dataPath, { recursive: true, force: true }) - - const archiveSize = fs.statSync(localPath + '.tar.lz4').size - const unarchiver = tar.extract(dataPath) - const decoder = lz4.createDecoderStream() - const input = fs.createReadStream(localPath + '.tar.lz4', streamOpts) - const progressStream = progress({}) - - extractProgressBar.start(archiveSize, 0, { - speed: 'N/A', - }) - - progressStream.on('progress', (progress) => { - extractProgressBar.update(progress.transferred, { - speed: bytesToMb(progress.speed).toFixed(2), - }) - }) - - await streamPipeline(input, progressStream, decoder, unarchiver) - .then(() => { - extractProgressBar.stop() - resolve() - }) - .catch((err) => { - extractProgressBar.stop() - reject(err.message) - }) - - const readMePath = `${dataPath}/ipfs/blocks/_README` - if (fs.existsSync(readMePath)) { - await fs.promises.rm(readMePath, {force: true}) - } - }) -} - -export const main = async (arg, config) => { - let time = Date.now() - - localPath = config.paths.archive - dataPath = config.paths.data - - const logTime = (message) => - console.log(`${message} ${(Date.now() - time) / 1000}s`) - - switch (arg) { - case 'load': { - await decompressToOutput().then(() => - logTime('Decompressed and copied in'), - ) - return - } - case 'compress': { - console.log('Compressing /data to archive.tar.lz4...') - await compressToArchive().then(() => logTime('Compressed archive in')) - return - } - case 'clean': { - console.log('Cleaning data directory...') - await fs.promises.rm(dataPath, { force: true, recursive: true }) - await fs.promises.mkdir(dataPath) - return - } - } -} diff --git a/packages/ens-test-env/src/index.d.ts b/packages/ens-test-env/src/index.d.ts deleted file mode 100644 index f934b01b..00000000 --- a/packages/ens-test-env/src/index.d.ts +++ /dev/null @@ -1 +0,0 @@ -export * from './config' diff --git a/packages/ens-test-env/src/index.js b/packages/ens-test-env/src/index.js deleted file mode 100755 index 3162d1e5..00000000 --- a/packages/ens-test-env/src/index.js +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env node - -/* eslint-disable */ - -import { Command, Option } from 'commander' -import path from 'node:path' -import { emitKeypressEvents } from 'node:readline' -import { main as fetchData } from './fetch-data.js' -import { main as manager } from './manager.js' -import { main as subgraph } from './subgraph.js' - -let config -const program = new Command() - -const __dirname = new URL('.', import.meta.url).pathname -const cwd = process.cwd() - -program - .name('ens-test-env') - .description('A testing environment for everything ENS') - .version(process.env.npm_package_version || '0.1.0') - .option('-c, --config ', 'Specify config directory') - .option('-a, --always-cleanup', 'Always cleanup after running') - .hook('preAction', async () => { - if (program.optsWithGlobals().alwaysCleanup) { - emitKeypressEvents(process.stdin) - if (process.stdin.isTTY) process.stdin.setRawMode(true) - process.stdin.on('keypress', (char, key) => { - if (key.ctrl && key.name === 'c') { - process.kill(process.pid, 'SIGINT') - } - }) - } - // if config arg supplied, get config path as next arg - const configDir = program.optsWithGlobals().config - // if config arg, try load config - if (configDir) { - try { - config = (await import(path.join(process.cwd(), configDir))).default - } catch { - program.error(`Config file ${configDir} not found`) - } - } else { - config = ( - await import(path.join(process.cwd(), 'ens-test-env.config.js')) - ).default - } - // if config doesn't have all data, throw error - if (!config) { - program.error('No valid config found') - return help() - } - // add default paths to config, and let them be replaced by specified vars - const paths = { - data: path.resolve(cwd, './data'), - archive: path.resolve(cwd, './archive'), - composeFile: path.resolve(__dirname, './docker-compose.yml'), - } - const configPaths = config.paths || {} - for (const [key, value] of Object.entries(configPaths)) { - if (typeof value === 'string') { - paths[key] = path.resolve(cwd, value) - } - } - config.paths = paths - }) - -program - .command('start') - .description('Starts the test environment') - .addOption(new Option('-nr, --no-reset', "Don't reset the data folder")) - .addOption( - new Option('-s, --save', 'Save data when exiting').implies({ - killGracefully: true, - verbosity: 1, - }), - ) - .addOption( - new Option( - '--extra-time